Source code for nvalchemi.training.distillation.strategy

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Knowledge-distillation strategy built on :class:`TrainingStrategy`."""

from __future__ import annotations

import dataclasses
import inspect
import warnings
from collections.abc import Iterable, Iterator, Mapping, Sequence, Sized
from contextlib import contextmanager, nullcontext
from typing import TYPE_CHECKING, Annotated, Any

import torch
from pydantic import Field, PrivateAttr, model_validator
from torch import distributed as dist

from nvalchemi._serialization import _dtype_deserialize, _import_cls
from nvalchemi._typing import Forces, ModelOutputs, NodePositions
from nvalchemi.data.datapipes.dataset import (
    BatchDatasetProtocol,
    dataset_device,
    same_device,
)
from nvalchemi.data.datapipes.samplers import distributed_shard
from nvalchemi.data.level_storage import resolve_device
from nvalchemi.distributed import collective_device
from nvalchemi.dynamics.sinks import HostMemory
from nvalchemi.dynamics.structure_sampler import WithinBudget
from nvalchemi.models._derivatives import HessianOperator
from nvalchemi.models.base import BaseModelMixin
from nvalchemi.training import TrainingStage
from nvalchemi.training import _spec_utils as strategy_spec
from nvalchemi.training import _strategy_validation as strategy_validation
from nvalchemi.training.distillation._attach import _attach_teacher_labels
from nvalchemi.training.distillation.config import (
    OnPolicyConfig,
    ResizableSink,
    _check_sole_migrator,
    _check_structure_status,
    _propagator_tree,
    _status_migrators,
)
from nvalchemi.training.distillation.hooks import (
    TeacherLabelHook,
    _ConvergedFrameHook,
    _DivergenceHook,
    _run_local_keys,
    _score_and_attach,
    _strip_replay_frame,
    _TrajectoryAgeHook,
    nonfinite_divergence,
)
from nvalchemi.training.distillation.losses.distribution import BoltzmannMatchingLoss
from nvalchemi.training.distillation.losses.embedding import (
    _PROJECTOR_REMEDY,
    EmbeddingMatchingLoss,
)
from nvalchemi.training.distillation.losses.hessian import HessianMatchingLoss
from nvalchemi.training.distillation.replay import (
    _SCHEMA_REMEDY,
    ReplayBuffer,
    _frame_schema,
    build_mixed_loader,
)
from nvalchemi.training.distillation.scoring import (
    _EMBEDDING_KEYS,
    _HVP_PROBE_FIELD,
    _TEACHER_FIELD_PREFIX,
    BUILTIN_SIGNALS,
    InProcessTeacherScorer,
    _node_embedding_shapes,
    _reject_foreign_fields,
    _restore_grad_flags,
    _snapshot_grad_flags,
    scorer_fields,
    signal_fields,
    signal_for_field,
)
from nvalchemi.training.distillation.seeding import (
    InitialStructures,
    InitialStructuresSource,
)
from nvalchemi.training.distributed import (
    all_reduce,
    all_reduce_flags,
    get_rank,
    get_world_size,
)
from nvalchemi.training.hooks.ddp import DDPHook
from nvalchemi.training.losses.composition import loss_target_keys
from nvalchemi.training.runtime import (
    eval_configured_models,
    evaluating,
    freeze_unconfigured_models,
    move_to_devices,
    train_configured_models,
    unwrap_model,
)
from nvalchemi.training.strategy import TrainingStrategy

if TYPE_CHECKING:
    from pathlib import Path

    from torch.optim.lr_scheduler import LRScheduler

    from nvalchemi.data.batch import Batch
    from nvalchemi.dynamics.base import BaseDynamics
    from nvalchemi.dynamics.sinks import DataSink
    from nvalchemi.hooks import TrainContext
    from nvalchemi.training import ValidationConfig
    from nvalchemi.training.losses.composition import (
        BaseLossFunction,
        ComposedLossFunction,
    )

__all__ = [
    "DistillationStrategy",
    "default_distillation_fn",
    "embedding_distillation_fn",
    "hessian_distillation_fn",
]

_REQUIRED_MODELS = frozenset({"student", "teacher"})
"""Model names every distillation strategy must be given."""

_PROJECTOR_MODEL = "projector"
"""Name of the auxiliary model that projects the student's embeddings."""

_PREDICTION_KEY_PREFIX = "predicted_"
"""Prefix the stock training functions publish every student output under."""

_HVP_OUTPUT = "hvp"
"""Student output name a Hessian objective's prediction key resolves to."""

_SEED_OK, _SEED_EMPTY, _SEED_FAILED = 0, 1, 2
"""Verdict codes a rank reports on its initial batch: seeded, empty shard, or failed build."""


def _is_synchronizing_wrapper(module: torch.nn.Module) -> bool:
    """Return whether ``module`` is a wrapper known to all-reduce its gradients.

    Recognizes :class:`~torch.nn.parallel.DistributedDataParallel` and, where
    the installed torch ships it, ``torch.distributed.fsdp``'s
    ``FullyShardedDataParallel``. Owning a ``module`` attribute is not enough:
    a hand-rolled wrapper exposes one without reducing anything. FSDP2's
    ``fully_shard`` works in place and leaves no wrapper object to recognize.
    """
    wrappers: tuple[type[torch.nn.Module], ...] = (
        torch.nn.parallel.DistributedDataParallel,
    )
    try:
        from torch.distributed.fsdp import FullyShardedDataParallel
    except ImportError:
        pass
    else:
        wrappers = (*wrappers, FullyShardedDataParallel)
    return isinstance(module, wrappers)


[docs] def default_distillation_fn( models: Mapping[str, BaseModelMixin], batch: Batch ) -> dict[str, torch.Tensor]: """Run the student forward pass and prefix output keys with ``predicted_``. The teacher is never called here: teacher knowledge reaches the loss as ``teacher_*`` batch fields, either written offline by :func:`~nvalchemi.training.distillation.label_dataset` or attached to the batch by :meth:`DistillationStrategy.attach_teacher_labels`. Parameters ---------- models : Mapping[str, BaseModelMixin] Named models of the strategy; only ``"student"`` is read. batch : Batch Input batch of atomic graphs. Returns ------- dict[str, torch.Tensor] Predictions keyed by ``predicted_<output_name>`` with ``None`` outputs omitted. """ outputs: ModelOutputs = models["student"](batch) return { f"{_PREDICTION_KEY_PREFIX}{key}": value for key, value in outputs.items() if value is not None }
[docs] def embedding_distillation_fn( models: Mapping[str, BaseModelMixin], batch: Batch ) -> dict[str, torch.Tensor]: """Run the student forward pass and add its node embeddings as a prediction. The student's node embeddings come from :meth:`~nvalchemi.models.base.BaseModelMixin.compute_embeddings`, a second pass over the batch, which is why :class:`~nvalchemi.training.distillation.EmbeddingMatchingLoss` needs this training function. The batch is left as it was found. When the strategy has a ``"projector"`` model, the projector is applied to the student's embeddings before they are returned, never to the teacher's, and it trains through its own ``optimizer_configs`` entry. Parameters ---------- models : Mapping[str, BaseModelMixin] Named models of the strategy; ``"student"`` and, when present, ``"projector"`` are read. batch : Batch Input batch of atomic graphs. Returns ------- dict[str, torch.Tensor] The stock ``predicted_*`` outputs plus ``predicted_node_embeddings``. Raises ------ RuntimeError If the student's ``compute_embeddings`` writes no ``node_embeddings``. Also if, while gradients are enabled and at least one student parameter is trainable, it writes embeddings detached from those parameters, as a wrapper that computes them under :func:`torch.no_grad` does; the embedding term could not train the student through them. A student with no trainable parameter skips this second check, since there is nothing for the embeddings to be detached from. A projector that declares ``frozen_student=True`` waives it too, for a student whose representation is frozen on purpose so that the projector alone carries the term. See Also -------- nvalchemi.training.distillation.EmbeddingProjector : The width adapter. Notes ----- The student runs twice per batch, which doubles its share of a training step. ``compute_embeddings`` is not part of the interface a :class:`~torch.nn.parallel.DistributedDataParallel` replica proxies, so the embedding pass runs on the wrapped module. Its gradients are still reduced. However, ``find_unused_parameters=True`` cannot see a student submodule exercised *only* by ``compute_embeddings``; that combination is the one configuration to avoid. """ predictions = default_distillation_fn(models, batch) student = unwrap_model(models["student"]) with batch.without_keys(*_EMBEDDING_KEYS): student.compute_embeddings(batch) if "node_embeddings" not in batch: raise RuntimeError( "Student compute_embeddings() must write ``node_embeddings`` onto " "the batch for embedding matching; got a batch carrying " f"{sorted(key for key in _EMBEDDING_KEYS if key in batch)!r}." ) embeddings = batch["node_embeddings"] projector = models.get(_PROJECTOR_MODEL) if ( torch.is_grad_enabled() and not embeddings.requires_grad and any(parameter.requires_grad for parameter in student.parameters()) and not ( projector is not None and getattr(unwrap_model(projector), "frozen_student", False) ) ): raise RuntimeError( "Student compute_embeddings() returned node embeddings detached from " f"the student's trainable parameters; got a {type(student).__name__!r} " "student whose embedding pass runs without gradients. The embedding " "objective would train the projector and nothing else. Override " "compute_embeddings on the wrapper so it runs with gradients enabled, " "register the projector with frozen_student=True if the student's " "trunk is frozen on purpose and the projector alone is to carry the " "term, or drop the term." ) if projector is not None: embeddings = projector(embeddings) predictions["predicted_node_embeddings"] = embeddings return predictions
[docs] def hessian_distillation_fn( models: Mapping[str, BaseModelMixin], batch: Batch ) -> dict[str, torch.Tensor]: """Run the student forward pass and add its Hessian-vector product. The student's Hessian-vector product is the product of its energy Hessian with the probe direction in ``teacher_hvp_probe``, the direction the teacher's own product was labeled with. It is taken on a second pass narrowed to the student's energy. The second pass is needed because a conservative model derives its forces from the autograd graph that the second derivative needs, and frees that graph outside training mode, so the stock forward's energy cannot be differentiated again. Both derivatives are taken with ``create_graph=True``, so :class:`~nvalchemi.training.distillation.HessianMatchingLoss` can backpropagate through them. The batch's ``requires_grad`` flags are restored, and the narrowed pass reuses the neighbor list the stock forward just ran on. Parameters ---------- models : Mapping[str, BaseModelMixin] Named models of the strategy; only ``"student"`` is read. batch : Batch Input batch, carrying the ``teacher_hvp_probe`` field the ``hessian`` teacher signal writes. Returns ------- dict[str, torch.Tensor] The stock ``predicted_*`` outputs plus ``predicted_hvp``. Raises ------ KeyError If the batch carries no probe, which means it was never labeled with the ``hessian`` signal, or if the student computes no energy to differentiate. Notes ----- On every frame the student trains on, the second pass adds the three backward passes :class:`~nvalchemi.models.HessianOperator` takes, the last of them through a second-order graph held for the whole step. A stochastic student draws afresh in the narrowed pass, so its curvature is measured on a different realization than its energy. """ probe = getattr(batch, _HVP_PROBE_FIELD, None) if probe is None: raise KeyError( f"Batch is missing the {_HVP_PROBE_FIELD!r} field required to take " "the student's Hessian-vector product along the direction the " "teacher was labeled with. Request the 'hessian' teacher signal so " "the probe travels with the label." ) student = unwrap_model(models["student"]) grad_flags = _snapshot_grad_flags(batch, student.model_config) try: predictions = default_distillation_fn(models, batch) predictions["predicted_hvp"] = _student_hvp(student, batch, probe) finally: _restore_grad_flags(batch, grad_flags) return predictions
def _student_hvp(student: BaseModelMixin, batch: Batch, probe: NodePositions) -> Forces: """Return the student's Hessian-vector product from an energy-only pass. Whatever neighbor list the batch carries is used as it stands. The teacher side isolates a rebuild because the batch it is handed was built for a different model at a different cutoff. This pass, however, runs the model the stock forward just ran on this very batch. A rebuild would only reproduce the list that forward already consumed, once per optimizer step, for every neighbor-list student. """ with student.narrowed_outputs({"energy"}), torch.enable_grad(): positions = batch.positions positions.requires_grad_(True) energy = student(batch).get("energy") if energy is None: raise KeyError( "Hessian matching differentiates the student's energy " "twice, so the student must compute an energy; got a " f"student declaring outputs {sorted(student.model_config.outputs)!r}." ) with HessianOperator.from_energy(energy, positions) as op: return op.matvec(probe, create_graph=True) _STOCK_TRAINING_FNS = { default_distillation_fn: frozenset(), embedding_distillation_fn: frozenset({"node_embeddings"}), hessian_distillation_fn: frozenset({_HVP_OUTPUT}), } """Predictions each stock training function adds beyond the student's outputs.""" def _derived_teacher_signals(loss_fn: ComposedLossFunction) -> frozenset[str]: """Return the built-in teacher signals the loss composition's targets require. A ``teacher_*`` target no built-in signal populates is a custom teacher field — one :func:`~nvalchemi.training.distillation.label_dataset` persisted from a custom scorer — that the batch must already carry, so it is passed over here rather than refused. A companion field that a signal writes beside its main field, such as the ``hessian`` probe direction, is refused instead. It records how the label was produced and is not a quantity to supervise against. """ signals: set[str] = set() for key in loss_target_keys(loss_fn): signal = ( signal_for_field(key) if key.startswith(_TEACHER_FIELD_PREFIX) else None ) if signal is None: continue produced = BUILTIN_SIGNALS[signal].field if key != produced: raise ValueError( f"Loss target {key!r} is a companion field: the {signal!r} signal " f"writes it beside {produced!r} to record how that field was " "produced. A probe is the direction the product was taken along, " "not a quantity the student is supervised against. Point the loss " f"at {produced!r}." ) signals.add(signal) return frozenset(signals) def _set_rebuild_overrides( strategy_cls: type[DistillationStrategy], overrides: Mapping[str, Any] ) -> dict[str, Any]: """Return the *overrides* that are set, checked against *strategy_cls*'s signature. A subclass overriding ``from_spec_dict`` with the base signature of an earlier release knows nothing of a later optional keyword, so an unset override is dropped instead of forwarded and a plain rebuild keeps working. A set one the subclass cannot take is refused here, where the keyword and the class can be named, rather than as a bare ``TypeError`` from the call. """ parameters = inspect.signature(strategy_cls.from_spec_dict).parameters takes_var_keyword = any( parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values() ) forwarded: dict[str, Any] = {} for name, value in overrides.items(): if value is None: continue if not takes_var_keyword and name not in parameters: raise TypeError( f"from_spec_dict: {strategy_cls.__name__}.from_spec_dict does not " f"accept the {name!r} keyword, so the supplied {value!r} cannot be " f"applied; add {name} to its signature or drop the override." ) forwarded[name] = value return forwarded def _matching_components( loss_fn: ComposedLossFunction, kind: type[Any] ) -> tuple[str, ...]: """Return the class names of the loss components that are instances of *kind*.""" return tuple( type(component).__name__ for component in loss_fn.components if isinstance(component, kind) ) @dataclasses.dataclass(frozen=True) class _RelaxationLifecycle: """Machinery a relaxation segment loop drives between its segments.""" capture: _ConvergedFrameHook structures: InitialStructures divergence: _DivergenceHook age: _TrajectoryAgeHook | None = None @contextmanager def _relaxation_lifecycle( config: OnPolicyConfig, state: Batch, label_hook: TeacherLabelHook | None = None ) -> Iterator[_RelaxationLifecycle | None]: """Install the convergence machinery of a relaxation run on the propagator. The config's :attr:`~OnPolicyConfig.convergence_criterion` is installed on the propagator in two roles. As a registered ``AFTER_STEP`` hook, it migrates the status of converged graphs, which freezes them. The propagator reports that migration at ``ON_GRADUATE``, where the capture hook stores the frame, and the segment boundary reads the migrated status. As the propagator's ``convergence_hook``, it is the detector that ends a chunk early once every graph has converged. A detector the propagator was built with is restored on exit. The criterion must be the only status migrator. A looser migrator would graduate a structure before this criterion accepts it, and neither capture route would store it. The config rejects such a migrator at construction, and the check runs again here for a hook registered since then. The lifecycle must also be the only source of refills, because a mid-segment refill compacts the surviving graphs and invalidates the divergence hook's per-graph record. A graph diverges when the config's :attr:`~OnPolicyConfig.divergence` predicate flags it; by default, when its state is no longer finite. A divergence hook registered after the criterion calls the predicate once per step, freezes each flagged graph at ``exit_status`` without capturing it, and records the verdict. The capture hook and the segment boundary read that record and never call the predicate themselves. The segment boundary then retires a diverged graph like a converged one. With ``quarantine_diverged`` the hook also zeroes the flagged graph's velocities and forces and makes its positions finite, so the frozen graph the propagator keeps evaluating stays a finite input to the model. A ``max_trajectory_steps`` lifetime installs an age hook after the divergence hook, so a graph is judged for divergence while it still moves. It freezes a graph at ``exit_status`` on the step it reaches the lifetime, so the graph graduates, is captured, retired, and backfilled exactly like a converged one. The lifetime alone is enough to manage a lifecycle: a thermostatted run with no criterion still retires and backfills, with the propagator's own detector left in place. Parameters ---------- config : OnPolicyConfig Segment-loop configuration, holding the criterion or the lifetime. state : Batch Initial batch, already carrying the bookkeeping :meth:`~nvalchemi.training.distillation.InitialStructures.initial_batch` stamped on it. label_hook : TeacherLabelHook | None, optional Labeling hook of the run's path route. The capture hook asks it which graphs it stored and skips a graph whose final frame it stored on the step the graph graduated. Default ``None``. Yields ------ _RelaxationLifecycle | None The machinery the segment loop drives, or ``None`` for a config that manages no lifecycle. Raises ------ ValueError If the propagator already carries a status-migrating criterion, if it carries a sampler of its own, or if the configured criterion migrates off a status no initial structure carries. """ criterion = config.convergence_criterion if not config.manages_lifecycle: yield None return dynamics = config.dynamics if criterion is not None: _check_sole_migrator(dynamics, criterion) if dynamics.sampler is not None: raise ValueError( "The relaxation lifecycle owns the refill as well as graduation, " "so the propagator must carry no sampler of its own; got " f"{type(dynamics.sampler).__name__!r}. A propagator that refills " "inside run reorders the batch mid-segment, so the capture hook " "would store the wrong structures and lose the minima. Give " "OnPolicyConfig.initial_structures the same budget instead, which " "backfills from the same dataset at the segment boundary, and " "leave the propagator's sampler unset." ) if criterion is not None: _check_structure_status(state, criterion) age = ( None if config.max_trajectory_steps is None else _TrajectoryAgeHook(config.max_trajectory_steps) ) divergence = _DivergenceHook( config.divergence or nonfinite_divergence, quarantine=config.quarantine_diverged, ) capture = _ConvergedFrameHook( sink=HostMemory(capacity=state.num_graphs), divergence=divergence, path=label_hook, ) detector = dynamics.convergence_hook # Registered ahead of the labeling hook, so a graph that converges, ages # out, or diverges on this step graduates before the path route reads its # status: no converged graph is stored twice, and no diverged one at all. # Divergence runs ahead of the age, so a graph that diverges on the step # its lifetime ends is still judged while moving and quarantined. installed = [ hook for hook in (criterion, divergence, age, capture) if hook is not None ] for hook in installed: dynamics.register_hook(hook) if criterion is not None: dynamics.convergence_hook = criterion try: yield _RelaxationLifecycle( capture=capture, structures=config.initial_structures, divergence=divergence, age=age, ) finally: dynamics.convergence_hook = detector for hook in installed: dynamics.hooks.remove(hook) def _lifetime_room( structures: InitialStructuresSource, state: Batch, survivors: torch.Tensor ) -> tuple[int | None, WithinBudget | None]: """Return the draw bounds of a lifetime backfill: the declared budget less the survivors. A source that declares no budget was propagated whole, so the room is the number of graphs that retired, as in a criterion-only lifecycle. Parameters ---------- structures : InitialStructuresSource Source the backfill draws from; its ``max_atoms``, ``max_edges``, and ``max_batch_size`` are read when it declares them. state : Batch Batch the segment ended on. survivors : torch.Tensor Boolean mask of the graphs that stay in the batch. Returns ------- tuple[int | None, WithinBudget | None] The ``limit`` and ``fits`` arguments of the draw. """ max_atoms = getattr(structures, "max_atoms", None) max_edges = getattr(structures, "max_edges", None) max_batch_size = getattr(structures, "max_batch_size", None) if max_atoms is None and max_edges is None and max_batch_size is None: return int((~survivors).sum()), None kept = int(survivors.sum()) edges_per_graph = state.num_edges_per_graph return ( None if max_batch_size is None else max(max_batch_size - kept, 0), WithinBudget( atoms=None if max_atoms is None else max(max_atoms - int(state.num_nodes_per_graph[survivors].sum()), 0), edges=None if max_edges is None or edges_per_graph.numel() == 0 else max(max_edges - int(edges_per_graph[survivors].sum()), 0), ), ) @contextmanager def _rank_local_propagator_seed(dynamics: BaseDynamics, offset: int) -> Iterator[None]: """Temporarily move a stochastic propagator's RNG onto this rank's own stream. A counter-based thermostat draws its noise from ``seed + step_count`` and the atom index. Without an offset, ranks stepping in lockstep would apply the same kicks to their different structures, and identical kicks to replicas of one geometry. :meth:`~nvalchemi.dynamics.BaseDynamics.seed_offset` moves every ``random_seed`` in the composition, because a ``FIRE2(...) + NVTLangevin(...)`` root exposes no seed of its own. The seeds are integers, so the negated offset on exit restores them exactly. Parameters ---------- dynamics : BaseDynamics Propagator whose ``random_seed``, and every sub-stage's, is offset on entry and restored on exit. Randomness the offset cannot reach is left untouched here and is reported by :meth:`DistillationStrategy._warn_shared_propagator_streams`. offset : int Amount added to every seed found. Zero leaves the propagator untouched. Yields ------ None Control while the propagator draws from this rank's stream. """ dynamics.seed_offset(offset) try: yield finally: dynamics.seed_offset(-offset) def _structure_count(structures: InitialStructuresSource) -> int | None: """Return how many rows *structures* holds, or ``None`` if it reports no count. A source that publishes its dataset, as :class:`~nvalchemi.training.distillation.InitialStructures` does, is counted through that dataset. Another source is counted through ``len()`` when it defines one. A streaming source with neither is left uncounted, and the world-size checks that need the count are skipped for it. """ dataset = getattr(structures, "dataset", None) if dataset is not None: return len(dataset) return len(structures) if isinstance(structures, Sized) else None def _propagates_student(propagator_model: object, student: BaseModelMixin) -> bool: """Return whether *propagator_model* is *student* or a model composing it.""" if propagator_model is student: return True modules = getattr(propagator_model, "modules", None) return callable(modules) and any(module is student for module in modules()) def _student_label_dtype(student: BaseModelMixin) -> torch.dtype | None: """Return the dtype teacher labels are cast to for *student*. The first floating-point parameter decides, floored at single precision: a store reads labels back at the dataset's ``positions`` dtype, float32 for essentially every dataset, so a narrower on-the-fly label would disagree with the persisted one. A student exposing no parameters gets ``None``, which keeps the teacher's own dtype. """ parameters = getattr(student, "parameters", None) if not callable(parameters): return None for parameter in parameters(): if parameter.is_floating_point(): if parameter.dtype.itemsize < torch.float32.itemsize: return torch.float32 return parameter.dtype return None def _segment_sink(config: OnPolicyConfig, num_graphs: int) -> DataSink: """Return the sink that stages one segment's labeled frames. A segment captures at most one frame per trajectory per labeled step, including the forced last frame. The sink must therefore hold ``(generation_steps + 1) * num_graphs`` frames. Without a configured ``capture_sink``, a host-memory sink of that capacity is built. A configured sink is kept. When it is too small, it is grown through ``resize(capacity)`` if it satisfies :class:`~nvalchemi.training.distillation.ResizableSink`. Parameters ---------- config : OnPolicyConfig Segment-loop configuration, holding the optional ``capture_sink``. num_graphs : int Number of trajectories the segment propagates. Returns ------- DataSink Empty sink of at least the segment's capacity. Raises ------ ValueError If the configured sink still holds frames, which the segment boundary would drain into the replay buffer as generated frames, or if it is too small and is not a ``ResizableSink``. """ capacity = (config.generation_steps + 1) * num_graphs sink = config.capture_sink if sink is None: return HostMemory(capacity=capacity) if len(sink) > 0: raise ValueError( "OnPolicyConfig.capture_sink must be empty when a segment starts, " "because everything it holds is drained into the replay buffer at " f"the segment boundary as generated frames; got {len(sink)!r} frames " f"in a {type(sink).__name__}. Drain or zero it first." ) if sink.capacity < capacity: if not isinstance(sink, ResizableSink): raise ValueError( "OnPolicyConfig.capture_sink must hold every frame one segment " "can capture, (generation_steps + 1) per trajectory: " f"{capacity!r} for {num_graphs!r} trajectories over " f"{config.generation_steps!r} steps; got a {type(sink).__name__} " f"of capacity {sink.capacity!r} without a resize method. Build it " "with at least that capacity, or give it resize(capacity)." ) sink.resize(capacity) return sink def _to_device(batch: Batch, device: torch.device) -> Batch: """Return *batch* on *device*, copying without blocking only into device memory. A non-blocking copy into host memory returns before the transfer completes. The moved batch's ``segment_lengths`` and ``batch_ptr`` are read on the host right away, so a copy into host memory blocks. """ return batch.to(device, non_blocking=device.type != "cpu") class _TeacherLabelHook: """Label the batch a forward pass is about to consume, training or validation. The first batch the seam labels raises one :class:`UserWarning` per strategy naming the missing fields, since from then on every such batch costs a teacher pass that a store written by :func:`~nvalchemi.training.distillation.label_dataset` would have spared. """ frequency = 1 stage = TrainingStage.BEFORE_FORWARD def __call__(self, ctx: TrainContext, stage: TrainingStage) -> None: # noqa: ARG002 """Attach the teacher fields the upcoming batch is missing.""" strategy: DistillationStrategy = ctx.workflow if ctx.batch is None or not strategy.label_missing: return missing = strategy._missing_teacher_fields(ctx.batch) if not missing: return strategy.attach_teacher_labels(ctx.batch) if strategy._warned_label_seam: return strategy._warned_label_seam = True warnings.warn( "DistillationStrategy is labeling batches on the fly: a batch reached " f"the forward pass without the teacher fields {missing!r}, so a teacher " "pass now runs for every such batch, in training and validation alike. " "Label the dataset ahead of time with label_dataset to avoid the cost, " "or set label_missing=False to surface it as a missing target instead.", UserWarning, stacklevel=2, )
[docs] class DistillationStrategy(TrainingStrategy): """Train a student against a frozen teacher's signals. A :class:`~nvalchemi.training.TrainingStrategy` over the named models ``"student"`` and ``"teacher"``. The teacher is frozen by omission — it must not appear in ``optimizer_configs`` — while the student and any auxiliary model must be configured with one. Teacher knowledge reaches the loss as ``teacher_*`` batch fields, so a built-in term distills by pointing its ``target_key`` at one (``EnergyMSELoss(target_key="teacher_energy")``) and :class:`~nvalchemi.training.distillation.AtomicEnergyMatchingLoss` reads ``teacher_atomic_energies``; mixing teacher and reference targets is ordinary loss composition. The signals the teacher is asked for are derived from the ``teacher_*`` targets the training loss and a ``validation_config`` loss read, or named in ``teacher_signals``, which must cover the derived set. They are checked against the teacher's outputs at construction, as are both losses' prediction keys against the outputs the student actually computes whenever the effective training or validation function is the stock :func:`default_distillation_fn`. Nothing re-runs on assignment, so pass ``validation_config`` to the constructor or, when rebuilding from a spec that excludes it, to :meth:`from_spec_dict`. Every resolved signal is required on every batch: a batch missing any resolved field is labeled on the fly by an internal ``BEFORE_FORWARD`` hook, in training and validation alike, unless ``label_missing=False`` leaves it to surface as a missing loss target, while a store written by :func:`~nvalchemi.training.distillation.label_dataset` with the same signal set trains with no teacher pass at all. A ``teacher_*`` target no built-in signal populates is a custom field such a store carries: it is neither derived nor attached, and a batch lacking it fails as a missing target. See :ref:`training-distillation-api` for the full contract. Setting ``on_policy`` switches :meth:`run` to the on-policy segment loop instead. In each segment, the student's own propagator generates frames and the teacher labels them. The labeled frames accumulate in a replay buffer. The segment then trains on a mixture of that buffer and ``reference_dataset`` at the configured ``replay_ratio``. The propagator holds the very module the optimizer updates, so every segment generates from a fresher policy than the last. That is what makes the data on-policy, and it is why the propagator's model is checked for object identity with ``models["student"]`` at construction. For a relaxation propagator, set ``OnPolicyConfig.fmax``. Converged structures are then stored once, graduate out of the batch at the segment boundary, and are replaced by fresh initial structures. The buffer therefore keeps filling with structures that are still moving. Beyond the signals that have a supervised shape, three objectives need more from the run than a target field. Embedding matching compares the student's and teacher's per-atom representations. It needs a second pass over the batch on both sides. Across architectures, it also needs the learnable :class:`~nvalchemi.training.distillation.EmbeddingProjector`, registered as a ``"projector"`` model with an optimizer of its own. Hessian matching compares Hessian-vector products, the products of each model's energy Hessian with a random probe direction. It needs the student's energy differentiated twice along the probe the teacher was labeled with. Neither of these predictions is a forward-pass output, so each objective needs the training function that produces it: :func:`~nvalchemi.training.distillation.embedding_distillation_fn` or :func:`~nvalchemi.training.distillation.hessian_distillation_fn`. Boltzmann matching compares the two models' Boltzmann distributions over configurations. It needs no new prediction, but it does need the on-policy loop, because it reads a batch as a sample of the student's own ensemble. All three requirements are checked at construction. Raises ------ ValueError If ``models`` is not a named mapping containing ``"student"`` and ``"teacher"``, if the teacher is given an optimizer config, if the student or an auxiliary model is not, if a loss component reads a prediction the student does not compute or names one outside the ``predicted_`` namespace under a stock ``training_fn``, if an explicit ``teacher_signals`` omits a signal a loss needs, if no built-in teacher signal is requested at all, if the teacher cannot produce a requested signal, if the teacher is a composition that plans more than one neighbor-list source, or if ``label_dtype`` is not a floating-point dtype. With an embedding objective under the stock embedding training function, also if the student publishes no node-embedding shape, if the student, projector, and teacher widths do not compose, or if the projector declares ``frozen_student=True`` over a fully trainable student. With a Hessian objective, also if the student computes no energy. With a Boltzmann objective, also if the run is not on-policy, if ``on_policy.samples_equilibrium`` is ``False``, if it is left ``None`` and the propagator relaxes or converges graphs out, if the term sits in the validation loss, or if a ``validation_config`` has no ``loss_fn`` of its own and would reuse the training loss. In on-policy mode, also if the run is sized in epochs, if the propagator holds neither the student nor a model composing it, if ``replay_ratio`` and ``reference_dataset`` disagree (a ratio below ``1`` requires a reference dataset and a ratio of ``1`` rejects one), if ``reference_dataset`` is empty or emits on an accelerator the run does not train on, if ``replay_device`` or the reference dataset's fields cannot be mixed with generated frames, or if the propagator's scorer and the reference dataset do not carry the same teacher fields. Examples -------- Distill energies, forces, and the teacher's per-atom energy decomposition from a store written by :func:`label_dataset`: >>> import torch >>> from nvalchemi.training import EnergyMSELoss, ForceMSELoss, OptimizerConfig >>> from nvalchemi.training.distillation import ( ... DistillationStrategy, ... AtomicEnergyMatchingLoss, ... ) >>> loss_fn = ( ... EnergyMSELoss(target_key="teacher_energy") ... + ForceMSELoss(target_key="teacher_forces", normalize_by_atom_count=True) ... + 0.1 * AtomicEnergyMatchingLoss() ... ) >>> strategy = DistillationStrategy( # doctest: +SKIP ... models={"student": student, "teacher": teacher}, ... optimizer_configs={ ... "student": [OptimizerConfig(optimizer_cls=torch.optim.Adam)] ... }, ... loss_fn=loss_fn, ... num_steps=1_000, ... ) >>> strategy.run(labeled_loader) # doctest: +SKIP Notes ----- Label precision is the scorer's decision. The strategy's own scorer is an :class:`~nvalchemi.training.distillation.InProcessTeacherScorer`, which disables autocast. It casts labels to ``label_dtype`` when one is given. By default the label dtype is inferred as the student's first floating-point parameter dtype, never below single precision, so a ``bfloat16`` or ``float16`` student gets float32 labels and needs ``dtype_policy="prediction_to_target"`` on its loss terms; a float64 student reads float32 back from a store and needs a ``dtype_policy`` too. Set ``label_dtype`` explicitly when the inference guesses wrong, as for a mixed-precision student whose first parameter is not representative or a student exposing no parameters at all, which otherwise keeps the teacher's own dtype. Labels are attached to the device-placed copy the strategy trains on, not the caller's batch, so a loader replaying the same systems costs one teacher pass per epoch, and the first batch labeled this way raises one :class:`UserWarning` naming the fields it lacked. Teacher conservativeness is not validated: a teacher predicting forces from its own head is first class, since every signal is detached before the student sees it. A teacher composition planning more than one neighbor-list source is refused, as by :class:`~nvalchemi.training.distillation.InProcessTeacherScorer`. The labeling hook is never serialized; :meth:`from_spec_dict` and :meth:`load_checkpoint` re-register it ahead of the caller's hooks and replace any carried copy. ``ValidationConfig(use_ema="auto")`` validates the EMA student against the live teacher (``model_source="mixed"``), while ``use_ema="always"`` also demands an inference-slot entry for the teacher. Checkpoints serialize every model, teacher included, so size the checkpoint interval for a large teacher. ``on_policy`` and ``reference_dataset`` hold live runtime objects that no spec can describe. :meth:`to_spec_dict` therefore omits them and warns. A strategy rebuilt from such a spec runs offline until they are supplied again. :meth:`from_spec_dict`, :meth:`from_checkpoint_dict`, and :meth:`load_checkpoint` accept them, together with the ``models`` the propagator holds. A strategy with a Boltzmann term refuses to rebuild without the loop, so for it the re-supply is mandatory. """ teacher_signals: Annotated[ frozenset[str] | None, Field( description=( "Teacher signals produced for every scored batch. ``None`` " "derives them from the ``teacher_*`` targets the training and " "validation losses read; an explicit set must cover those and " "may request more, at the cost of re-scoring every batch a " "store labeled without the extra fields delivers." ) ), ] = None label_missing: Annotated[ bool, Field( description=( "Whether a batch lacking the required ``teacher_*`` fields is " "labeled on the fly by a teacher forward pass, in training and " "validation alike. ``False`` skips the teacher, so an unlabeled " "batch surfaces as a missing loss target." ) ), ] = True label_dtype: Annotated[ torch.dtype | None, Field( description=( "Floating-point dtype teacher labels are cast to when a batch is " "labeled on the fly. ``None`` infers it from the student's first " "floating-point parameter, never below float32; an explicit dtype " "is passed to the teacher scorer verbatim." ) ), ] = None on_policy: Annotated[ OnPolicyConfig | None, Field( default=None, exclude=True, description=( "Segment-loop configuration turning ``run`` into on-policy " "distillation. ``None`` keeps the offline loop over the " "dataloader the caller passes to ``run``." ), ), ] = None reference_dataset: Annotated[ BatchDatasetProtocol | None, Field( default=None, exclude=True, description=( "Teacher-labeled dataset the on-policy mixture draws " "its ``1 - replay_ratio`` share from. Required whenever the " "ratio is below 1, and read only in on-policy mode." ), ), ] = None _scorer: InProcessTeacherScorer | None = PrivateAttr(default=None) _teacher_fields: tuple[str, ...] = PrivateAttr(default=()) _warned_label_seam: bool = PrivateAttr(default=False) _warned_unwrapped_student: bool = PrivateAttr(default=False) _replay_buffer: ReplayBuffer | None = PrivateAttr(default=None) _validation_probe_index: int | None = PrivateAttr(default=None) _validated_step: int | None = PrivateAttr(default=None) @property def replay_buffer(self) -> ReplayBuffer | None: """Frames generated so far, or ``None`` before an on-policy run starts. One buffer serves every :meth:`run` call on a strategy. A run continued with a larger ``num_steps`` therefore keeps training on everything generated so far instead of discarding and regenerating it. The trajectory is still reseeded on each call. """ return self._replay_buffer @property def teacher_scorer(self) -> InProcessTeacherScorer: """Scorer producing the resolved teacher signals for one batch.""" if self._scorer is None: raise RuntimeError( "DistillationStrategy has no teacher scorer; it is built during " "validation and must not be cleared." ) return self._scorer @property def structure_shard(self) -> tuple[int, ...]: """Rows of the initial structures this rank propagates from. Once :meth:`run` has installed this rank's shard on the source, the rows are read from the source, which publishes them with the ``rank`` and ``world_size`` they were dealt for, as :class:`~nvalchemi.dynamics.OrderedStructureSampler` does. Before that, they are dealt here from the launcher's rank and world size through the same :func:`~nvalchemi.data.datapipes.distributed_shard` the sampler uses. The property therefore means the same thing before and after a run. Returns ------- tuple[int, ...] Indices into ``on_policy.initial_structures.dataset``, in dataset order. Empty for an offline strategy, and for a source that publishes neither its rows nor a count. See Also -------- nvalchemi.dynamics.OrderedStructureSampler.shard : How the rows are dealt, and the shard-local position the deal resets. """ if self.on_policy is None: return () structures = self.on_policy.initial_structures rank = get_rank(self.distributed_manager) world_size = get_world_size(self.distributed_manager) installed = ( getattr(structures, "rank", None), getattr(structures, "world_size", None), ) if installed == (rank, world_size): return tuple(getattr(structures, "rows", ())) total = _structure_count(structures) if total is None: return () return tuple( distributed_shard( list(range(total)), num_replicas=world_size, rank=rank, drop_last=False, pad=False, ) ) @model_validator(mode="before") @classmethod def _default_distillation_training_fn(cls, data: Any) -> Any: """Fall back to the stock student-forward training function.""" if not isinstance(data, dict): return data normalized = dict(data) if normalized.get("training_fn") is None: normalized["training_fn"] = default_distillation_fn return normalized @model_validator(mode="before") @classmethod def _prepend_labeling_hook(cls, data: Any) -> Any: """Put the internal teacher-labeling hook ahead of the caller's hooks. A seam carried in the incoming hooks is replaced rather than kept, so rebuilding a strategy from a live one's ``hooks`` leaves exactly one labeling hook, still ahead of every caller hook. """ if not isinstance(data, dict): return data normalized = dict(data) normalized["hooks"] = [ _TeacherLabelHook(), *( hook for hook in (normalized.get("hooks") or []) if not isinstance(hook, _TeacherLabelHook) ), ] return normalized @model_validator(mode="after") def _validate_distillation(self) -> DistillationStrategy: """Enforce the student/teacher contract and resolve the teacher signals.""" missing_models = _REQUIRED_MODELS - set(self.models) if self.single_model_input or missing_models: raise ValueError( "DistillationStrategy needs a named-model mapping holding " f"'student' and 'teacher'; got models={sorted(self.models)!r}." ) if "teacher" in self.optimizer_configs: raise ValueError( "The teacher is frozen by omission, so it must not appear in " f"optimizer_configs; got {sorted(self.optimizer_configs)!r}." ) unconfigured = set(self.models) - set(self.optimizer_configs) - {"teacher"} if unconfigured: raise ValueError( "Every model but the teacher must be given an optimizer config; " f"got unconfigured {sorted(unconfigured)!r}." ) if self.label_dtype is not None and not self.label_dtype.is_floating_point: raise ValueError( "label_dtype must be a floating-point dtype or None; got " f"{self.label_dtype!r}." ) self._validate_student_outputs() signals = self._resolve_teacher_signals() self._scorer = InProcessTeacherScorer( self.models["teacher"], signals, dtype=( _student_label_dtype(self.models["student"]) if self.label_dtype is None else self.label_dtype ), ) self._teacher_fields = signal_fields(signals) return self def _validate_student_outputs(self) -> None: """Check both losses' prediction keys against what the student computes. The check runs only under a stock training function. Each one emits ``active_outputs`` intersected with ``outputs``, plus whatever it derives on top, so a narrowed student is caught here rather than on its first batch. The validation loss is checked whenever its effective function, ``validation_fn`` falling back to ``training_fn``, is a stock one. """ derived = _STOCK_TRAINING_FNS.get(self.training_fn) if derived is not None: self._validate_prediction_keys(self.loss_fn.components, "training", derived) validation = self.validation_config if validation is None or validation.loss_fn is None: return validation_derived = _STOCK_TRAINING_FNS.get( validation.validation_fn or self.training_fn ) if validation_derived is not None: self._validate_prediction_keys( validation.loss_fn.components, "validation", validation_derived ) def _validate_prediction_keys( self, components: Sequence[BaseLossFunction], side: str, derived: frozenset[str], ) -> None: """Check one composition's prediction keys, naming *side* in every error. *derived* names the predictions the stock training function in play adds on top of the student's own outputs. """ student = self.models["student"] declared = student.model_config.outputs active = student.output_data() for component in components: key = getattr(component, "prediction_key", None) if key is None: continue label = f"{side} loss component {type(component).__name__!r}" if not key.startswith(_PREDICTION_KEY_PREFIX): raise ValueError( f"The {label} reads prediction_key={key!r}, which the stock " "training functions never emit: they publish every student " f"output under {_PREDICTION_KEY_PREFIX}<output>. Rename the " "key into that namespace, or pass a training_fn that owns " "its own convention." ) output = key.removeprefix(_PREDICTION_KEY_PREFIX) if output in active or output in derived: continue if output in _EMBEDDING_KEYS: raise ValueError( f"The {label} reads prediction_key={key!r}, which this " "training_fn cannot produce: embeddings come from the " "student's compute_embeddings(), not from its forward pass. " "Pass training_fn=embedding_distillation_fn, which calls " "compute_embeddings, routes the result through a 'projector' " f"model when one is registered, and returns it under {key!r}." ) if output == _HVP_OUTPUT: raise ValueError( f"The {label} reads prediction_key={key!r}, which this " "training_fn cannot produce: a Hessian-vector product is a " "second derivative of the student's energy, not a forward " "output. Pass training_fn=hessian_distillation_fn, which " "differentiates the energy twice along the probe the teacher " "was labeled with." ) if output in declared: raise ValueError( "Student declares but does not compute the output required by " f"the {label} reading prediction_key={key!r}; got " f"active_outputs={sorted(active)!r}, missing {output!r}. Add " "it to the student's model_config.active_outputs." ) raise ValueError( f"Student cannot produce the output required by the {label} " f"reading prediction_key={key!r}; got outputs={sorted(declared)!r}, " f"missing {output!r}." ) def _resolve_teacher_signals(self) -> frozenset[str]: """Return the signals both losses need, widened by an explicit request.""" derived = {"training": _derived_teacher_signals(self.loss_fn)} validation = self.validation_config if validation is not None and validation.loss_fn is not None: derived["validation"] = _derived_teacher_signals(validation.loss_fn) required: frozenset[str] = frozenset().union(*derived.values()) resolved = required if self.teacher_signals is None else self.teacher_signals uncovered = { side: sorted(signals - resolved) for side, signals in derived.items() if signals - resolved } if uncovered: raise ValueError( "teacher_signals must cover every teacher target the training and " f"validation losses read; got {sorted(resolved)!r}, missing " f"{uncovered!r}." ) if not resolved: raise ValueError( "DistillationStrategy needs at least one teacher signal; got no " "built-in teacher_* target in the training or validation loss and " f"teacher_signals={self.teacher_signals!r}. A custom teacher_* field " "the batch already carries is not a signal; name one in " "teacher_signals or read a built-in teacher target." ) return resolved @model_validator(mode="after") def _validate_on_policy(self) -> DistillationStrategy: """Enforce the segment loop's duration, ownership, and mixture contract.""" if self.on_policy is None: if self.reference_dataset is not None: raise ValueError( "reference_dataset is the on-policy mixture's reference share " "and is read only by the segment loop; got it set alongside " "on_policy=None. Offline distillation trains on the " "dataloader passed to run()." ) return self if self.num_steps is None: raise ValueError( "On-policy distillation is sized in optimizer steps: every " "segment builds its own loader, so there is no fixed epoch to " f"convert. Got num_epochs={self.num_epochs!r}; set num_steps " "instead." ) propagator_model = getattr(self.on_policy.dynamics, "model", None) if not _propagates_student(propagator_model, self.models["student"]): held = ( "no model at all" if propagator_model is None else f"a separate {type(propagator_model).__name__} instance" ) raise ValueError( "OnPolicyConfig.dynamics must propagate the very module " "registered as models['student'], on its own or composed into " "a larger model: the data is on-policy only because each " "optimizer step is immediately visible to the propagator. Got " f"a propagator holding {held}; build the dynamics around the " "student object itself." ) if self.on_policy.replay_ratio < 1.0 and self.reference_dataset is None: raise ValueError( "A replay_ratio below 1 mixes reference data into every batch, " f"so reference_dataset is required; got replay_ratio=" f"{self.on_policy.replay_ratio!r} and reference_dataset=None." ) if self.on_policy.replay_ratio == 1.0 and self.reference_dataset is not None: raise ValueError( "replay_ratio=1 draws every sample of every batch from the " "replay buffer, so a reference dataset would only be checked for " "schema and device and never sampled; got replay_ratio=1.0 " f"alongside a {type(self.reference_dataset).__name__} " "reference_dataset. Drop reference_dataset, or lower replay_ratio " "to mix it in." ) if self.reference_dataset is not None and len(self.reference_dataset) == 0: raise ValueError( "A replay_ratio below 1 draws part of every batch from " "reference_dataset, so it has to hold at least one sample; got an " f"empty {type(self.reference_dataset).__name__}. Pass the labeled " "reference set the mixture draws from, or set replay_ratio=1 to " "train on generated frames alone." ) # One probe batch serves the device and schema checks alike. probe = ( None if self.reference_dataset is None else self.reference_dataset.load_batches([[0]])[0] ) self._validate_reference_device(probe) self._validate_mixture_device(probe) self._validate_reference_schema(probe) self._validate_generation_signals() return self def _validate_reference_device(self, probe: Batch | None) -> None: """Reject a reference dataset emitting on an accelerator the run does not train on. Parameters ---------- probe : Batch | None One batch already drawn from ``reference_dataset``. A composed dataset, or a store opened without a device, is measured by the device of this batch. ``None`` when there is no reference dataset. """ if self.reference_dataset is None: return reference_device = dataset_device(self.reference_dataset, probe) primary = self.devices[0] if reference_device.type == "cpu" or same_device(reference_device, primary): return raise ValueError( "A segment's mixture is collated on the reference dataset's own " "device before the strategy moves it, so a reference dataset that " "emits on an accelerator has to emit on the device the run trains " f"on; got a reference dataset emitting on {reference_device!s} and " f"devices[0]={primary!s}. A Zarr-backed Dataset resolves an unset " "device to CUDA whenever one is visible — open it as " f"Dataset(..., device={str(primary)!r}) to follow the run, or leave " "it in host memory." ) def _validate_mixture_device(self, probe: Batch | None) -> None: """Reject a staging device the reference dataset cannot be collated with. Parameters ---------- probe : Batch | None One batch already drawn from ``reference_dataset``. The device of a composed dataset or of a store opened without a device is read from it. ``None`` when there is no reference dataset to measure. """ if self.reference_dataset is None or self.on_policy.replay_device is None: return reference_device = dataset_device(self.reference_dataset, probe) replay_device = torch.device(self.on_policy.replay_device) if same_device(reference_device, replay_device): return raise ValueError( "A mixed batch is collated before the strategy moves it, so the " "replay buffer and reference_dataset have to live on one device; " f"got replay_device={replay_device!s} and a reference dataset " f"emitting on {reference_device!s}. Leave replay_device unset to " "stage generated frames wherever the reference dataset lives, or " f"load the reference dataset on {replay_device!s}." ) def _validate_reference_schema(self, probe: Batch | None) -> None: """Reject a reference dataset holding fields no generated frame can carry. The full schema comparison runs inside the first segment's :func:`~nvalchemi.training.distillation.build_mixed_loader`, after a whole generation phase has been paid for. The part that depends on nothing the run produces is checked here: the predictions, neighbor tensors, and bookkeeping that the labeling hook strips from every frame. In particular, this catches a store that :func:`~nvalchemi.training.distillation.label_dataset` wrote over a reference set, which keeps that set's ``energy`` and ``forces``. """ if probe is None: return dropped = _run_local_keys(self.on_policy.dynamics) unmixable = sorted( name for name in _frame_schema(probe) if name.partition(".")[2] in dropped ) if not unmixable: return raise ValueError( "reference_dataset carries fields no generated frame can, so the " "mixture would be rejected on the first segment's loader; got " f"{unmixable!r} on reference_dataset, which the labeling hook strips " f"from every frame it stores. {_SCHEMA_REMEDY}" ) def _validate_generation_signals(self) -> None: """Check the propagator's teacher fields against the reference dataset and loss. A scorer that declares neither ``label_fields`` nor a set of built-in signals writes fields that cannot be known before it scores a batch. Both checks are then skipped with a warning. Running them against an empty set would reject a custom scorer that in fact produces exactly what the reference dataset carries. """ generated = scorer_fields(self.on_policy.teacher_scorer) if generated is None: warnings.warn( "The propagator's scorer declares neither label_fields nor " "built-in signals, so the teacher fields it writes are unknown " "until the first segment has generated them: neither their " "parity with reference_dataset nor whether every generated " "frame is scored twice can be checked at construction, and a " "mismatch surfaces as a rejected mixture once a whole " "generation phase has been paid for. Got signals=" f"{sorted(self.on_policy.teacher_scorer.signals)!r}; declare " "label_fields on the scorer to restore both checks.", UserWarning, stacklevel=2, ) return _reject_foreign_fields(generated, "A scorer's label_fields") if self.reference_dataset is not None: stored = frozenset( field for field in self.reference_dataset.field_names if field.startswith(_TEACHER_FIELD_PREFIX) ) if frozenset(generated) != stored: raise ValueError( "Generated frames and reference_dataset must carry the same " "teacher fields, because mixing them into one batch keeps " f"only the fields both hold; got generation " f"{sorted(generated)!r} and reference {sorted(stored)!r}. " "Request the same signals on OnPolicyConfig.teacher_scorer, " "or relabel the reference dataset with label_dataset." ) self._warn_on_partial_generation_signals(generated) def _warn_on_partial_generation_signals(self, generated: tuple[str, ...]) -> None: """Warn when generated frames will be relabeled on their way into training. The comparison is by field rather than by signal name. A custom scorer that writes the fields the loss reads, under signal names of its own, is therefore not reported as missing them. """ missing = frozenset(self._teacher_fields) - frozenset(generated) if missing: warnings.warn( "The propagator's scorer does not produce every teacher field " "the loss reads, so each generated frame is scored twice: once " "during generation and again on its way into a training step; " f"missing {sorted(missing)!r}. Request the signals populating " "those fields on OnPolicyConfig.teacher_scorer to pay the " "teacher once.", UserWarning, stacklevel=2, ) @model_validator(mode="after") def _validate_advanced_objectives(self) -> DistillationStrategy: """Enforce what the embedding, Hessian, and Boltzmann terms need.""" self._validate_embedding_matching() self._validate_hessian_matching() self._validate_distribution_matching() return self def _matching_sides( self, kind: type[Any], training_fn: Any = None ) -> list[tuple[tuple[str, ...], str]]: """Return the components of *kind* each side's loss runs under *training_fn*. A ``validation_config`` carrying its own ``loss_fn`` reaches the student through its effective validation function, ``validation_fn`` falling back to ``training_fn``. A term that only the validation loss holds is therefore checked here too, and every message names the side the term came from. ``training_fn=None`` matches a term whatever function its side runs. """ sides = [(self.loss_fn, self.training_fn, "training")] validation = self.validation_config if validation is not None and validation.loss_fn is not None: sides.append( ( validation.loss_fn, validation.validation_fn or self.training_fn, "validation", ) ) return [ (terms, side) for loss_fn, effective_fn, side in sides if (terms := _matching_components(loss_fn, kind)) and (training_fn is None or effective_fn is training_fn) ] def _validate_embedding_matching(self) -> None: """Check that the student, projector, and teacher embedding widths compose. Only the stock embedding training function is checked, because it is the one whose routing this check can reason about. It projects the student's embeddings with ``models['projector']`` when there is one, so the widths have to compose. A caller's own training function owns its own routing. """ student = self.models["student"] for terms, side in self._matching_sides( EmbeddingMatchingLoss, embedding_distillation_fn ): label = f"{side} loss component(s) {list(terms)!r}" student_shape = _node_embedding_shapes(student).get("node_embeddings") if student_shape is None: raise ValueError( f"The {label} match the student's node embeddings, so the " "student must publish a 'node_embeddings' shape and write it " "in compute_embeddings(); got embedding_shapes=" f"{sorted(_node_embedding_shapes(student))!r}." ) width = student_shape[-1] projector = ( self.models[_PROJECTOR_MODEL] if _PROJECTOR_MODEL in self.models else None ) if projector is not None: in_features = getattr(projector, "in_features", None) if in_features is not None and in_features != width: raise ValueError( "The projector reads the student's embeddings, so its input " f"width must be the student's; got in_features={in_features!r} " f"against a student of width {width!r}." ) if getattr(projector, "frozen_student", False) and all( parameter.requires_grad for parameter in student.parameters() ): raise ValueError( "The projector declares frozen_student=True, so the " f"{label} are to train it alone over a frozen student " "representation, but every student parameter is trainable. " "Freeze the student's trunk with requires_grad_(False), or " "drop the flag. The flag also waives the check that catches " "embeddings detached by accident, so it must not stay set " "on a trainable student." ) width = getattr(projector, "out_features", width) teacher_shape = _node_embedding_shapes(self.models["teacher"]).get( "node_embeddings" ) if teacher_shape is not None and width != teacher_shape[-1]: raise ValueError( f"The {label} compare representations component by component, " "so what reaches the loss must have the teacher's width; got " f"{width!r} against a teacher of width {teacher_shape[-1]!r}. " f"{_PROJECTOR_REMEDY}" ) def _validate_hessian_matching(self) -> None: """Check that the student computes the energy a Hessian term differentiates. A direct-force student triggers a warning rather than a refusal. The term is computable and its values are right, but the second derivative it drives down is the one the energy head implies. That is not the Jacobian of the force head a force loss trains, so the curvature that decides an integrator's stability for that student is left unsupervised. """ sides = self._matching_sides(HessianMatchingLoss, hessian_distillation_fn) if not sides: return active = self.models["student"].output_data() for terms, side in sides: if "energy" not in active: raise ValueError( f"The {side} loss component(s) {list(terms)!r} need the " "student's Hessian-vector product, which " "hessian_distillation_fn takes by differentiating the " "student's energy twice, so the student must compute an " f"energy; got active outputs {sorted(active)!r}." ) config = self.models["student"].model_config if "forces" in config.outputs and "forces" not in config.autograd_outputs: named = sorted({name for terms, _ in sides for name in terms}) warnings.warn( f"Loss component(s) {named!r} differentiate the student's energy " "twice, but the student predicts its forces with a head of its " "own rather than as that energy's gradient; got autograd_outputs=" f"{sorted(config.autograd_outputs)!r}. The curvature term then " "supervises the energy head alone. The force head a force loss " "trains gets no second-order signal at all. Distill a conservative " "student for the term to reach the forces, or read it as a " "constraint on the energy surface only.", UserWarning, stacklevel=2, ) def _validate_distribution_matching(self) -> None: """Require an equilibrium on-policy sample for every Boltzmann term. The estimator reads the batch as a sample of the student's own canonical distribution, so it needs the segment loop and a propagator that keeps sampling. ``samples_equilibrium`` declares whether the propagator does; when it is ``None``, :meth:`_refuse_inferred_non_equilibrium` infers the answer. What reaches the loss is a draw from the replay buffer, so an unbounded buffer and a ``replay_ratio`` below ``1`` each draw a warning. Validation data is off-policy by construction, so a term on the validation side is refused, and so is a validation config that would reuse the training loss. """ sides = { side: terms for terms, side in self._matching_sides(BoltzmannMatchingLoss) } if "validation" in sides: raise ValueError( f"The validation loss component(s) {list(sides['validation'])!r} " "read a batch as a sample of the student's own Boltzmann " "distribution, but a validation set is off-policy by construction. " "It is a fixed sample of whatever produced it, so the uniform " "weights the estimator assumes are wrong rather than noisy, and " "the metric would mislead checkpoint selection and the metric " "schedulers. Give the validation config a pointwise loss instead, " "such as EnergyMSELoss(target_key='teacher_energy') + " "ForceMSELoss(target_key='teacher_forces')." ) terms = sides.get("training") if terms is None: return if self.on_policy is None: raise ValueError( f"Loss component(s) {list(terms)!r} read a batch as a sample of " "the student's own Boltzmann distribution, over configurations " "the student itself visited; got on_policy=None. An offline " "dataset is a sample of whatever produced it, which makes the " "estimator's uniform weights wrong rather than noisy. Configure " "the segment loop, or drop the term. A spec or checkpoint carries " "no segment loop, so a rebuild re-supplies it: pass on_policy, " "with the models its propagator holds, to load_checkpoint or " "from_spec_dict, or restore_checkpoint into a strategy already " "built with it. Reweighting an off-policy sample is not offered, " "because the importance weights the estimator folds away as " "uniform cannot be recovered from the batch. An existing dataset " "reaches the term as reference_dataset instead, mixed into " "generated frames by replay_ratio and read as regularization." ) if self.on_policy.samples_equilibrium is False: raise ValueError( f"Loss component(s) {list(terms)!r} are defined on an equilibrium " "ensemble; got on_policy.samples_equilibrium=False, which declares " "that the propagator does not sample one. Generate with a " "propagator that does and declare it with samples_equilibrium=True, " "leave the setting None to infer it from the propagator, or drop " "the term." ) if self.on_policy.samples_equilibrium is None: self._refuse_inferred_non_equilibrium(terms) if self.on_policy.replay_ratio < 1.0: warnings.warn( f"Loss component(s) {list(terms)!r} read every batch as a sample " "of the student's own ensemble; got replay_ratio=" f"{self.on_policy.replay_ratio!r}, which mixes reference frames " "into each batch. The estimator cannot tell those frames from " "generated ones and weights them as if the student had visited " "them. Set replay_ratio=1 to keep reference rows out of the batch, " "or keep the reference share small and read the term as " "regularization. With replay_ratio=1 the estimate is as current as " "the replay buffer, which is what replay_capacity bounds.", UserWarning, stacklevel=2, ) if self.on_policy.replay_capacity is None: labelings = ( self.on_policy.generation_steps // self.on_policy.label_frequency + 1 ) warnings.warn( f"Loss component(s) {list(terms)!r} read every batch as a sample " "of the student's own ensemble; got replay_capacity=None. An " "unbounded replay buffer retires nothing, and every segment's " "loader draws uniformly over all of it. After N segments only " "about one N-th of a batch came from the current student, and the " "rest is the time-average of every policy the run has had, which " "is the off-policy sample an offline dataset is refused for. Bound " "the capacity to what one segment or a few segments yield, " f"{labelings} labeling(s) per segment here at one frame per " "walker each, and leave replay_eviction='fifo' so the stalest " "frames retire first. A one-segment buffer is the most current " "and gives the softmax the fewest distinct configurations to " "weight.", UserWarning, stacklevel=2, ) if ( self.validation_config is not None and self.validation_config.loss_fn is None ): raise ValueError( f"Loss component(s) {list(terms)!r} are defined on the batches " "the student generated; got validation_config.loss_fn=None. A " "ValidationConfig without a loss_fn of its own reuses this " "strategy's training loss, Boltzmann term included, and the " "labeling seam scores validation batches for it, so the term " "would run on the validation set. That set is off-policy by " "construction: it is a fixed sample of whatever produced it, its " "graphs need not be one system's configurations, and reducing " "energies by k_B T lets the term dominate the composite metric " "that checkpoint selection and the metric schedulers read. Give " "the validation config a pointwise loss, such as " "EnergyMSELoss(target_key='teacher_energy') + " "ForceMSELoss(target_key='teacher_forces'), or drop the " "validation config." ) def _refuse_inferred_non_equilibrium(self, terms: tuple[str, ...]) -> None: """Refuse a propagator whose parts show that it does not keep sampling. This is the rule ``samples_equilibrium=None`` stands for. A relaxation propagator, one whose class declares :attr:`~nvalchemi.dynamics.BaseDynamics.samples_equilibrium` false as the built-in optimizers do, descends to a minimum. A converging propagator freezes each graph as it converges, whether the criterion is the propagator's own, a hook registered on it, or the one the loop installs from ``fmax`` or ``convergence_hook``. Registered criteria come from the same walk the lifecycle uses, :func:`~nvalchemi.training.distillation.config._status_migrators`, kept when they graduate a graph past the root's ``exit_status``; a migration into a status the root still steps hands the graph to another sub-stage, as a :class:`~nvalchemi.dynamics.FusedStage` does between its own, and is not a stop. Every refusal names the declaration that overrides it. """ assert self.on_policy is not None # noqa: S101 # narrowing stages = list(_propagator_tree(self.on_policy.dynamics)) relaxing = [ type(stage).__name__ for stage in stages if not stage.samples_equilibrium ] if relaxing: raise ValueError( f"Loss component(s) {list(terms)!r} are defined on an equilibrium " f"ensemble, but the run has a relaxation propagator driving " f"{relaxing!r}; a relaxation does not sample one. It descends to a " "minimum, so its frames are a path rather than a distribution. " "Generate with a thermostatted integrator, drop the term, or set " "on_policy.samples_equilibrium=True if the propagator does sample " "an equilibrium ensemble." ) if ( self.on_policy.fmax is not None or self.on_policy.convergence_hook is not None ): configured = ( f"fmax={self.on_policy.fmax!r}" if self.on_policy.fmax is not None else f"convergence_hook={self.on_policy.convergence_hook!r}" ) raise ValueError( f"Loss component(s) {list(terms)!r} are defined on an equilibrium " f"ensemble, but the segment loop has {configured}, so it converges " "graphs out and stops sampling them. The criterion freezes each " "converged graph at the state it converged to and graduates it out " "of the batch the term is matching against. The loop installs the " "criterion on the propagator at run time, so it is checked here " "on the loop's settings rather than on the propagator. Generate " "without a convergence criterion, drop the term, or set " "on_policy.samples_equilibrium=True if the propagator does sample " "an equilibrium ensemble." ) exit_status = self.on_policy.dynamics.exit_status graduating = { id(hook) for hook in _status_migrators(self.on_policy.dynamics) if hook.target_status >= exit_status } converging = [ type(stage).__name__ for stage in stages if getattr(stage, "convergence_hook", None) is not None or any(id(hook) in graduating for hook in getattr(stage, "hooks", ())) ] if converging: raise ValueError( f"Loss component(s) {list(terms)!r} are defined on an equilibrium " f"ensemble, but a convergence hook on {converging!r} converges " "graphs out, so the propagator stops sampling them. Every " "converged graph is frozen at the state it converged to. Generate " "without a convergence hook, drop the term, or set " "on_policy.samples_equilibrium=True if the propagator does sample " "an equilibrium ensemble." ) def attach_teacher_labels(self, batch: Batch) -> bool: """Attach the teacher fields *batch* is missing, and report whether it did. A batch already carrying every resolved ``teacher_*`` field is left untouched, so pre-labeling a batch that later reaches :meth:`run` costs one teacher pass; a batch carrying only some of them is re-scored in full, since a partial set was written for a different signal set. The scorer is called inside whatever autocast region the training loop holds open. The strategy's own scorer disables autocast, so the labels match what :func:`~nvalchemi.training.distillation.label_dataset` persisted wherever the store returns the label dtype. Inside :meth:`validate`, the Hessian probe is drawn from a seed keyed to the batch's position in the pass, handed to the scorer for that call alone; see that method. Parameters ---------- batch : Batch Batch to label in place, already on the teacher's device. Returns ------- bool ``True`` when the teacher ran, ``False`` when *batch* already carried every resolved field. """ if not self._missing_teacher_fields(batch): return False index = self._validation_probe_index if index is not None: self._validation_probe_index = index + 1 _attach_teacher_labels( batch, self.teacher_scorer.label(batch, probe_seed=index) ) return True def _missing_teacher_fields(self, batch: Batch) -> list[str]: """Return the resolved teacher fields *batch* does not carry.""" return [field for field in self._teacher_fields if field not in batch] def validate(self) -> dict[str, Any] | None: """Run a validation pass whose batches keep their probe directions. Validation batches are relabeled on the fly on every pass. A Hessian term would then draw a fresh Hutchinson probe each time, which moves the reported number for a student that has not changed. Each batch is instead scored along a direction keyed to its position in the pass, passed to the scorer as a call-time ``probe_seed`` that leaves the scorer's own setting untouched. The metric is then a function of the student alone, as long as the validation data iterates in a stable order. Training keeps drawing fresh probes. Returns ------- dict[str, Any] | None The validation summary, also stored on ``last_validation``. """ self._validation_probe_index = 0 try: return super().validate() finally: self._validation_probe_index = None def run(self, dataloader: Iterable[Batch] | None = None) -> None: """Execute the offline training loop or the on-policy segment loop. Without ``on_policy``, this is :meth:`~nvalchemi.training.TrainingStrategy.run` over *dataloader*. With ``on_policy``, the strategy owns the loop and repeats three phases until ``num_steps`` optimizer steps have run. In the *generate* phase, the propagator advances the live state batch by ``generation_steps`` steps; on the first segment, the state is seeded from ``initial_structures``. In the *label and capture* phase, a :class:`~nvalchemi.training.distillation.TeacherLabelHook` registered on the propagator labels every ``label_frequency`` steps and the segment's last frame. It copies each labeled frame into the capture sink, which is drained into the replay buffer. The capture sink is host memory unless ``capture_sink`` names another. In the *train* phase, a freshly built mixed loader draws ``training_steps_per_segment`` batches at ``replay_ratio``, and each batch goes through the ordinary per-batch stages. For relaxation propagators, whose trajectories end, an ``OnPolicyConfig.fmax`` threshold adds a fourth phase between generation and training: *graduate and backfill*. Converged structures are stored once, as the minimum they reached. They then leave the batch and are replaced by fresh initial structures while the source still holds any. Generation stops once the source runs dry and the last trajectory finishes. The remaining steps train on the frames already in the buffer. Parameters ---------- dataloader : Iterable[Batch] | None, optional Batches to train on in offline mode. Default ``None``, which is required in on-policy mode and rejected otherwise. Raises ------ ValueError If *dataloader* is ``None`` in offline mode or supplied in on-policy mode, if a multi-rank launch has fewer initial structures than ranks or leaves the student unsynchronized, if a segment's loader produces no batches, if the propagator already carries a status-migrating criterion or a sampler of its own, or if the configured criterion migrates off a status no initial structure carries. Warns ----- UserWarning If a lifecycle-managed run runs out of trajectories and initial structures before reaching ``num_steps``. The remaining steps then train on the frames already generated. Also once per segment boundary that retires trajectories the ``divergence`` predicate flagged, by default those whose positions or forces stopped being finite. Also if a multi-rank run cannot deal its initial structures out in equal shares, or if its propagator holds randomness that the rank offsets cannot separate. Notes ----- One segment is one epoch. ``AFTER_EPOCH`` and epoch-cadence validation fire at segment boundaries, and step-cadence validation fires inside segments. The run closes with one final validation unless a cadence already validated at the final step. The segment is also the restart granularity. The propagator state is not checkpointed. A resumed run reseeds its trajectory, and a segment that a checkpoint interrupted is counted as finished on resume. A second call keeps the replay buffer the first call filled and reseeds only the trajectory. The student generates in evaluation mode and trains in training mode. A propagator model that merely composes the student is held in evaluation mode for the whole loop. Every mode is restored on exit. Generated frames are staged on ``reference_dataset``'s device unless ``replay_device`` names another. Splitting a built-in propagator's run into segments is exact, because ``run`` never resets ``step_count`` and the Langevin thermostat's generator is keyed on it. The progress of a segment that exits early is read from ``dynamics.step_count``. A :class:`~nvalchemi.dynamics.FusedStage` pays its priming forward pass once per segment. Across ranks, the loop is data-parallel. Each rank propagates its own strided shard of ``initial_structures``, labels the generated frames with its own teacher replica, and fills its own replay buffer. The reference dataset stays replicated, and every rank draws from all of it under a rank-offset ``seed``. The same offset moves every ``random_seed`` that the propagator and its sub-stages expose, through :meth:`~nvalchemi.dynamics.BaseDynamics.seed_offset`. A stage that holds randomness the offset cannot move is named in a warning. The student's gradient all-reduce, installed by a :class:`~nvalchemi.training.hooks.DDPHook`, is the only per-step training traffic between ranks. Setup adds small collectives, which check the shards and the replay placement, and validation all-reduces its metrics. The teacher never joins a collective. Every rank runs the same number of segments and batches, so the ranks reach each all-reduce together. A launch with fewer initial structures than ranks is refused before any segment runs, and so is one whose student nothing wraps, unless ``require_wrapped_student=False`` waives that check for a wrapper that works in place. See :ref:`training-distillation-api` for the mixture and schema contract and the scale-out runbook. Relaxation runs are the reason a segment can exit early, and ``OnPolicyConfig.fmax`` turns such a run into a lifecycle. For the duration of the loop, the criterion is registered ahead of the labeling hook and installed as the propagator's convergence detector. A converged structure is captured once, on the step its ``status`` reaches ``exit_status``, and left out of every later path capture. At the segment boundary, initial structures are drawn to fill the room the graduates freed. A budgeted :class:`~nvalchemi.training.distillation.InitialStructures` draws them from the rows it has not served yet. An unbudgeted one draws them only with ``recycle=True``; otherwise the batch narrows. Once no trajectory is left and no initial structure remains to start one, the loop warns and trains on the buffer it has until ``num_steps``. """ if self.on_policy is None: if dataloader is None: raise ValueError( "Offline distillation trains on the caller's batches; got " "run(dataloader=None) with on_policy=None. Pass a " "dataloader, or configure on_policy to generate one." ) super().run(dataloader) return if dataloader is not None: raise ValueError( "On-policy distillation builds its own loader every segment " "from reference_dataset and the replay buffer; got a " f"{type(dataloader).__name__} passed to run(). Set it as " "reference_dataset instead." ) self._run_on_policy(self.on_policy) def _run_on_policy(self, config: OnPolicyConfig) -> None: """Drive generate-label-train segments until ``num_steps`` is reached. The rank shard is installed on the initial structures here rather than at construction, for two reasons. The world size is set by the launcher. Installing the shard also resets the position, which a second ``run()`` on one strategy needs: that call keeps the replay buffer and reseeds only the trajectory, so the source must start again at the front of its shard. A propagator model that merely composes the student is moved whole to the generation device. Only the named models move with the strategy, so a correction held only by the composition would otherwise stay on the device it was built on. """ training_started = False strategy_context = nullcontext(self) if self._context_depth > 0 else self with strategy_context: self._prepare_setup_hooks() self._validate_runtime_devices() self._validate_structure_shards(config) self._warn_unequal_structure_shards(config) self._warn_shared_propagator_streams(config) self.models = move_to_devices(self.models, self.devices) propagator_model = config.dynamics.model if propagator_model is not self.models["student"] and isinstance( propagator_model, torch.nn.Module ): propagator_model.to(self.devices[0]) unsynchronized = self.models["student"] self._run_setup_hooks() self._validate_synchronized_student(config, unsynchronized) replay_device = self._resolve_replay_device(config) target_step_count = self._resolve_target_step_count(None) if self.step_count >= target_step_count: return self._close_interrupted_segment() self._apply_requires_grad_filter() try: primary_device = self.devices[0] flat_opts, flat_scheds = self._setup_runtime_optimizers( rebuild=not self._resume_optimizer_state ) config.initial_structures.shard( get_rank(self.distributed_manager), get_world_size(self.distributed_manager), ) state = self._seed_initial_state(config, primary_device) if self._replay_buffer is None: self._replay_buffer = ReplayBuffer( capacity=config.replay_capacity, eviction=config.replay_eviction, admission=config.replay_admission, device=replay_device, ) buffer = self._replay_buffer # Compare against the student module, not a DDPHook's wrapper. student = unwrap_model(self.models["student"]) propagator_model = config.dynamics.model held_propagator = ( evaluating(propagator_model) if isinstance(propagator_model, torch.nn.Module) and propagator_model is not student else nullcontext() ) # Only a lifecycle stores graduated graphs by another route; # a propagator that manages its own keeps every frame here. label_hook = TeacherLabelHook( config.teacher_scorer, frequency=config.label_frequency, exit_status=config.dynamics.exit_status if config.manages_lifecycle else None, ) with _relaxation_lifecycle(config, state, label_hook) as lifecycle: config.dynamics.register_hook(label_hook) try: # Freeze the teacher for both phases and keep the student in # eval mode outside the training phase. A composition holding # the student is not in models, so it is held separately. with ( freeze_unconfigured_models( self.models, self.optimizer_configs ), eval_configured_models(self.models, self.optimizer_configs), held_propagator, _rank_local_propagator_seed( config.dynamics, self._rank_seed_offset(config) ), ): while self.step_count < target_step_count: if state is not None: state = self._generate_segment( config, state, label_hook, lifecycle, buffer, target_step_count, ) training_steps = min( config.training_steps_per_segment, target_step_count - self.step_count, ) with train_configured_models( self.models, self.optimizer_configs ): training_started = self._train_segment( config, buffer, training_steps=training_steps, target_step_count=target_step_count, training_started=training_started, flat_opts=flat_opts, flat_scheds=flat_scheds, ) finally: config.dynamics.hooks.remove(label_hook) if self._last_batch is not None: self._update_hook_snapshot(loss_out=None) self._run_hooks(TrainingStage.AFTER_TRAINING, self._last_batch) if ( self.validation_config is not None and self._validated_step != self.step_count ): self.validate() self._step_metric_schedulers() finally: self._restore_requires_grad_filter() def _validate_structure_shards(self, config: OnPolicyConfig) -> None: """Reject initial structures too few to give every rank at least one. This is the pre-check every rank runs alone and reaches the same verdict on, before any collective: it needs only the structure count and the world size, which every rank reads identically. The sampler's own empty-shard refusal cannot play that part, because it is raised only on the rank whose shard came up empty, while its peers go on to the first collective and block there. A source that reports no count is checked later by :meth:`_seed_initial_state`, which reduces the verdict across the world instead. The world size is read at run time, after a launcher has initialized the process group. An offline strategy built by the same script distributes freely. Parameters ---------- config : OnPolicyConfig Configuration of the loop about to start. Raises ------ ValueError If the initial structures hold fewer rows than there are ranks. """ world_size = get_world_size(self.distributed_manager) if world_size == 1: return num_structures = _structure_count(config.initial_structures) if num_structures is None: return if num_structures < world_size: raise ValueError( "Every rank propagates its own share of the initial structures, " "so there has to be at least one for each; got " f"{num_structures!r} structures on {world_size!r} ranks. Start " "from more structures, or launch fewer ranks." ) def _seed_initial_state( self, config: OnPolicyConfig, device: torch.device ) -> Batch: """Draw this rank's first batch, and require every rank to have one. :meth:`_validate_structure_shards` can only check a source that reports how many rows it holds. A source that deals its own shards is checked here instead, by what its ``shard()`` actually left this rank. An empty shard surfaces as the ``ValueError`` a source raises when it has nothing to serve; any other exception the build raises is caught as well, because a rank that left before the exchange would leave its peers blocking in it. Each rank's verdict, a batch, an empty shard, or a failed build, is reduced across the world as one code per rank, the way :func:`~nvalchemi.training.distributed.all_reduce_flags` reduces a flag, before any rank reaches the first gradient collective. A rank whose build failed then raises its own exception, so the cause surfaces where it happened, and every other rank raises a refusal naming the ranks that stopped the run. Parameters ---------- config : OnPolicyConfig Configuration of the loop about to start, already sharded. device : torch.device Device the run trains on, which the seeded batch is moved to. Returns ------- Batch Initial batch of this rank's shard. Raises ------ ValueError If any rank's shard seeded nothing, and no rank's build failed for another reason. RuntimeError If another rank's build raised something other than an empty shard. That rank raises the exception itself. Exception Whatever this rank's own build raised, once the verdict has been exchanged. """ world_size = get_world_size(self.distributed_manager) if world_size == 1: return _to_device(config.initial_structures.initial_batch(), device) state: Batch | None = None failure: Exception | None = None try: state = _to_device(config.initial_structures.initial_batch(), device) except Exception as exc: failure = exc seeded = 0 if state is None else state.num_graphs if failure is not None and not isinstance(failure, ValueError): verdict = _SEED_FAILED elif seeded == 0: verdict = _SEED_EMPTY else: verdict = _SEED_OK verdicts = torch.zeros( world_size, dtype=torch.int64, device=collective_device() ) verdicts[get_rank(self.distributed_manager)] = verdict verdicts = all_reduce(verdicts, self.distributed_manager, op=dist.ReduceOp.MAX) if not bool(verdicts.any()): return state if verdict == _SEED_FAILED: failure.add_note( "Raised after the initial-batch verdict was exchanged across " f"{world_size!r} ranks; every rank stops here." ) raise failure failed = (verdicts == _SEED_FAILED).nonzero().flatten().tolist() empty = (verdicts == _SEED_EMPTY).nonzero().flatten().tolist() if failed: raise RuntimeError( f"Ranks {failed!r} of {world_size!r} failed to build an initial " "batch" + ( f", and ranks {empty!r} were dealt a shard that seeded nothing" if empty else "" ) + f"; this rank seeded {seeded!r} structures. Every rank stops " "here, before the first collective, and the failing ranks raise " "the cause. Fix it there and relaunch." ) raise ValueError( f"Ranks {empty!r} of {world_size!r} were dealt a shard that seeded " f"nothing; this rank seeded {seeded!r} structures. Every rank " "propagates its own shard of the initial structures, so there has to " "be at least one for each. Provide at least one structure for each " "rank: start from more structures, deal them out evenly in a source " "that shards itself, or launch fewer ranks." ) from failure def _warn_unequal_structure_shards(self, config: OnPolicyConfig) -> None: """Report initial structures the world cannot deal out in equal shares. Every rank draws the same number of replay samples per batch from a buffer that holds only its own trajectories, and DDP averages the gradients evenly. A frame from a shorter shard is therefore drawn more often. The check needs only the structure count and the world size, so every rank reaches the same verdict without a collective. Parameters ---------- config : OnPolicyConfig Configuration of the loop about to start. Warns ----- UserWarning If the initial structures do not divide evenly across the ranks. """ world_size = get_world_size(self.distributed_manager) if world_size == 1: return num_structures = _structure_count(config.initial_structures) if num_structures is None: return smallest, remainder = divmod(num_structures, world_size) if remainder == 0: return warnings.warn( "The initial structures do not divide evenly across the world: " f"{num_structures!r} structures dealt across {world_size!r} ranks " f"leave {smallest + 1!r} on {remainder!r} of them and {smallest!r} on " "the rest. Every rank draws the same number of replay samples per " "batch from a buffer holding only its own trajectories, and the " "gradients are averaged rank by rank, so a frame from a shorter " "shard reaches the optimizer with up to " f"{(smallest + 1) / smallest:.2f}x the weight of one from a longer " f"shard. Size the dataset as a whole multiple of {world_size!r} to " "weight every generated frame alike.", UserWarning, stacklevel=2, ) def _warn_shared_propagator_streams(self, config: OnPolicyConfig) -> None: """Report the propagator randomness that the rank offsets cannot separate. This warns rather than raises. A propagator that is deterministic in the stages :meth:`~nvalchemi.dynamics.BaseDynamics.seed_offset` cannot reach is correct, and nothing here can tell it apart from one that hides randomness there. A zero offset asks the engine what it cannot move without moving anything. The verdict concerns the whole world, not this rank alone, so rank zero reports it too. It is reported before any segment is generated or any teacher pass is paid for. Parameters ---------- config : OnPolicyConfig Configuration of the loop about to start. Warns ----- UserWarning If a stage holds randomness the offset cannot move, or if nothing in the composition exposes an integer ``random_seed`` at all. """ if get_world_size(self.distributed_manager) == 1: return unmoved = config.dynamics.seed_offset(0) seeded = any( isinstance(getattr(node, "random_seed", None), int) for node in _propagator_tree(config.dynamics) ) if unmoved: warnings.warn( "Part of this run's propagator stays on the shared random " f"stream: {sorted(set(unmoved))!r} hold randomness that " "seed_offset cannot move, a torch.Generator with no integer " "random_seed or a random_seed without a setter. Every rank " "draws the same stream in those stages. Ranks seeded with " "replicas of one structure then generate the same trajectories, " "which the teacher labels once per rank. Seed those generators " "from the global rank yourself, or expose the seed as a settable " "integer random_seed the loop can offset.", UserWarning, stacklevel=2, ) elif not seeded: warnings.warn( "This run's propagator noise could not be moved onto per-rank " f"streams: {type(config.dynamics).__name__!r} exposes no integer " "random_seed, and neither does anything it composes. A " "deterministic propagator has no stream to separate and can " "ignore this. One that keeps its randomness elsewhere has to be " "given a rank-distinct seed by the caller, or every rank applies " "the same noise to the structures it was dealt.", UserWarning, stacklevel=2, ) def _validate_synchronized_student( self, config: OnPolicyConfig, unsynchronized: BaseModelMixin ) -> None: """Reject a multi-rank run whose student nothing keeps in step. The on-policy loop needs this where plain data-parallel training does not: each rank generates its frames from its own copy of the student, so a student whose gradients are not synchronized drifts into a private replica on every rank, each rank trains and generates from a different policy, and only rank zero's is checkpointed. Called after the ``SETUP`` stage, which is where a :class:`~nvalchemi.training.hooks.DDPHook` replaces the models it is given with wrappers. When the strategy carries a ``DDPHook``, the check reads :attr:`~nvalchemi.training.hooks.DDPHook.wrapped_keys` and passes when ``"student"`` is among them. Failing that, with or without a ``DDPHook``, it passes when the stage put something else in the student's place and that object owns the student, as :func:`~nvalchemi.training.runtime.unwrap_model` reads ownership, so a hand-rolled or FSDP wrapper passes too, even beside a ``DDPHook`` that was given other models. A student the caller wrapped before ``run`` counts as synchronized only when the object handed over is a wrapper known to reduce gradients, as :func:`_is_synchronizing_wrapper` lists them: a ``DDPHook`` leaves such a model alone, so nothing replaces it for the stage to show, and ownership alone would let any module exposing a ``module`` attribute pass without synchronizing anything. The fallback otherwise compares against the module registered before the stage rather than only unwrapping the one registered afterwards; otherwise a bare student that happens to hold a submodule named ``module`` would pass as wrapped. The model the propagator holds plays no part in either path. A wrapper that works in place, or one this check does not recognize, leaves nothing to compare, so ``require_wrapped_student=False`` waives the check with a one-time warning, and gradient synchronization becomes the caller's responsibility. Parameters ---------- config : OnPolicyConfig Configuration whose ``require_wrapped_student`` governs the check. unsynchronized : BaseModelMixin The module registered as ``models["student"]`` before the ``SETUP`` stage ran. Raises ------ ValueError If nothing has taken ownership of the student to synchronize its gradients. Warns ----- UserWarning Once per strategy, when the check is waived on a multi-rank world. """ world_size = get_world_size(self.distributed_manager) if world_size == 1: return if not config.require_wrapped_student: if not self._warned_unwrapped_student: self._warned_unwrapped_student = True warnings.warn( "Install an in-place wrapper (FSDP2 fully_shard, hook-based " "gradient sync) on the student; synchronizing its gradients " "is the caller's responsibility. require_wrapped_student=False " "skips the check that the SETUP stage replaced " f"models['student'] with a wrapper owning it, on {world_size!r} " "ranks. Without a wrapper, every rank trains and generates " "from a private student, and only rank zero's is checkpointed.", UserWarning, stacklevel=2, ) return student = self.models["student"] ddp_hooks = [hook for hook in self.hooks if isinstance(hook, DDPHook)] wrapped = frozenset().union(*(hook.wrapped_keys for hook in ddp_hooks)) if "student" in wrapped: return if student is not unsynchronized and unwrap_model(student) is unsynchronized: return if student is unsynchronized and _is_synchronizing_wrapper(student): return if student is not unsynchronized: observed = ( f"a {type(student).__name__!r} that does not own the one handed over" ) elif unwrap_model(student) is not student: observed = ( "the same object that was handed over, a " f"{type(student).__name__!r} owning a module through a wrapper " "this check does not recognize" ) else: observed = "the same object that was handed over" if ddp_hooks: observed = ( "not among the models the DDPHook wrapped, which are " f"{sorted(wrapped)!r}, and {observed}" ) raise ValueError( f"After the SETUP stage, models['student'] is {observed}, on " f"{world_size!r} ranks. A multi-rank segment loop trains one student " "from every rank's own frames, so its gradients have to be " "synchronized. Without that, each rank trains and generates from a " "private student, and only rank zero's is checkpointed. Add a DDPHook " "to hooks, which wraps every optimizer-configured model at setup and " "leaves the frozen teacher out of the all-reduce, or set " "require_wrapped_student=False for a wrapper that works in place." ) def _close_interrupted_segment(self) -> None: """Count a segment that a restored run stopped partway through as finished. Each segment builds its own loader, so the batches an interrupted segment had left are lost with it, and the trajectory is reseeded anyway. Closing the segment keeps ``BEFORE_EPOCH`` firing for the resumed segment and keeps ``epoch_step_count`` inside the segment budget. It also moves the mixture sampler past the epoch index the interrupted segment drew with. The parent's :meth:`_prepare_epoch_step_count` does not fit here: it reconciles against a fixed number of batches per epoch, and a run that continues from offline epochs of another size has none. """ if self.epoch_step_count == 0: return self.epoch_count += 1 self.epoch_step_count = 0 self._refresh_hook_counters() def _validation_checkpoint(self, stage: TrainingStage) -> bool: """Run a scheduled validation and remember the step it fired at. The segment loop's closing validation is skipped when a cadence already validated at the final step, so a metric-driven scheduler is never stepped twice on one set of metrics. """ fired = super()._validation_checkpoint(stage) if fired: self._validated_step = self.step_count return fired def _train_segment( self, config: OnPolicyConfig, buffer: ReplayBuffer, *, training_steps: int, target_step_count: int, training_started: bool, flat_opts: list[torch.optim.Optimizer], flat_scheds: list[LRScheduler | None], ) -> bool: """Train one segment's mixture and close it as an epoch. Returns ------- bool Whether the ``BEFORE_TRAINING`` stage has fired by now. """ loader = build_mixed_loader( self.reference_dataset, buffer, replay_ratio=config.replay_ratio, batch_size=config.batch_size, num_batches=training_steps, seed=config.seed + self._rank_seed_offset(config), ) self._set_sampler_epoch(loader) primary_device = self.devices[0] consumed = 0 for batch in loader: if consumed >= training_steps or self.step_count >= target_step_count: break batch = _to_device(batch, primary_device) self._update_hook_snapshot(batch=batch, loss_out=None) if not training_started: self._run_hooks(TrainingStage.BEFORE_TRAINING, batch) training_started = True if self.epoch_step_count == 0: self._run_hooks(TrainingStage.BEFORE_EPOCH, batch) self._train_batch_with_optimizers(batch, flat_opts, flat_scheds) self._validation_checkpoint(TrainingStage.AFTER_OPTIMIZER_STEP) consumed += 1 if consumed == 0: raise ValueError( "The segment's mixed loader produced no batches before the " "target step count was reached; ensure reference_dataset and " "the replay buffer together hold at least one batch of " f"batch_size={config.batch_size!r} samples." ) self.epoch_count += 1 self.epoch_step_count = 0 self._refresh_hook_counters() self._run_hooks(TrainingStage.AFTER_EPOCH, self._last_batch) self._validation_checkpoint(TrainingStage.AFTER_EPOCH) return training_started def _generate_segment( self, config: OnPolicyConfig, state: Batch, label_hook: TeacherLabelHook, lifecycle: _RelaxationLifecycle | None, buffer: ReplayBuffer, target_step_count: int, ) -> Batch | None: """Propagate one segment, store what it produced, and refill the batch. Parameters ---------- config : OnPolicyConfig Segment-loop configuration. state : Batch Batch this segment propagates from. label_hook : TeacherLabelHook Hook labeling and capturing the frames along the path. lifecycle : _RelaxationLifecycle | None Convergence machinery, or ``None`` when none is managed. buffer : ReplayBuffer Buffer the segment's frames are stored in. target_step_count : int Training step the run ends at, named by the exhaustion warning. Returns ------- Batch | None The batch the next segment propagates from, or ``None`` once every trajectory has finished and no initial structure is left to start a fresh one from. That case warns once. """ # Sized per segment because a refill changes the trajectory count. The # converged route keeps its own host sink, one frame per graph. label_hook.sink = _segment_sink(config, state.num_graphs) if lifecycle is not None: lifecycle.capture.sink = HostMemory(capacity=state.num_graphs) state = config.dynamics.run(state, n_steps=config.generation_steps) self._capture_segment(config, state, label_hook, buffer) if lifecycle is None: return state self._capture_converged(config, lifecycle, buffer) refilled = self._backfill_segment(config, lifecycle, state) if refilled is None: self._warn_generation_exhausted(config, target_step_count, state) return refilled def _capture_converged( self, config: OnPolicyConfig, lifecycle: _RelaxationLifecycle, buffer: ReplayBuffer, ) -> None: """Label the structures that converged this segment and store them. Converged frames are captured without teacher labels, at the step each structure reached its minimum. Here the teacher labels all of the segment's graduates in one pass, under the same guards the path route uses. The frames are then stripped to the replay-frame contract, so they match the schema the path frames froze the buffer with. Finally they are moved to the buffer's own device. """ sink = lifecycle.capture.sink if len(sink) == 0: return frames = _to_device(sink.drain(), self.devices[0]) _score_and_attach(config.teacher_scorer, frames) buffer.extend( _strip_replay_frame(frames, config.dynamics).to(buffer.device or "cpu") ) def _backfill_segment( self, config: OnPolicyConfig, lifecycle: _RelaxationLifecycle, state: Batch, ) -> Batch | None: """Graduate the finished structures and backfill fresh ones in their place. A trajectory finishes in one of three ways. It converges and the criterion freezes it, it reaches ``max_trajectory_steps`` and the age hook freezes it, or it diverges and the lifecycle freezes it on the step the divergence predicate flagged it. This method counts the diverged ones from the divergence hook's record and warns about them. It then draws initial structures to fill the room the graduates freed: at most as many structures as graduated, within the atoms they held. The draw is also bounded by their edges, but only when the source declared ``max_edges``, because a dataset's stored edge count is not the neighbor list a propagator rebuilds. A criterion-only lifecycle passes over a row that does not fit, so one oversized structure cannot block every refill behind it. A lifecycle with ``max_trajectory_steps`` draws differently: its room is the source's declared budget less what the survivors hold, since a room measured by the retired graphs alone would shrink with every under-packed draw, and it stops at the first row that does not fit. Every trajectory retires in turn, so the room grows to the full budget and the row is served then, and no row of the source is lost to a scan that would otherwise run through the shard once the room is full. The propagator's per-structure state follows the membership change. The run restamps its bookkeeping over the appended rows and keeps only the ``system_id`` the source assigned. A structure stored with the ``status`` it once graduated on therefore still moves. Returns ------- Batch | None The refilled batch, or ``None`` once nothing is left to propagate. """ dynamics = config.dynamics status = state["status"].view(-1)[: state.num_graphs] graduated = status >= dynamics.exit_status if not bool(graduated.any()): return state flagged_graphs = lifecycle.divergence.diverged diverged = ( 0 if flagged_graphs is None else int((graduated & flagged_graphs).sum()) ) if diverged: predicate = lifecycle.divergence.divergence flagged = ( "their positions or forces stopped being finite" if predicate is nonfinite_divergence else f"the divergence predicate {predicate!r} flagged them" ) warnings.warn( f"{diverged} of {state.num_graphs} generated trajectories " f"diverged: {flagged}. The lifecycle froze them on that step " "without storing that frame. They are now dropped from the " "batch and replaced from the initial structures. A diverging " "student is extrapolating; shorten the propagator's step, or " "register a MaxForceClampHook on it.", UserWarning, stacklevel=2, ) structures = lifecycle.structures edges_per_graph = state.num_edges_per_graph if lifecycle.age is None: limit: int | None = int(graduated.sum()) fits = WithinBudget( atoms=int(state.num_nodes_per_graph[graduated].sum()), edges=int(edges_per_graph[graduated].sum()) if getattr(structures, "max_edges", None) is not None and edges_per_graph.numel() > 0 else None, ) else: limit, fits = _lifetime_room(structures, state, ~graduated) fresh = structures.draw( limit=limit, fits=fits, # Under a lifetime every trajectory retires, so the room grows to # the full budget in turn and no row needs passing over; skipping # would burn the shard once the room is full. InitialStructures # still passes over a row larger than the whole budget. on_miss="skip" if lifecycle.age is None else "stop", ) lifecycle.divergence.reset() survivors = torch.where(~graduated)[0] if lifecycle.age is not None: lifecycle.age.retain(survivors, len(fresh)) refilled = state.index_select(survivors) if survivors.numel() > 0 else None if fresh: appended = type(state).from_data_list(fresh, device=state.device) if refilled is None: refilled = appended else: refilled.append(appended) # As refill_check does: the state rows follow the membership change, # and stale converged indices must not reach the next step's context. dynamics._sync_state_to_batch( survivors, len(fresh), state if refilled is None else refilled ) dynamics._last_converged = None if refilled is None: return None # Appending keeps only the keys both sides hold, so every bookkeeping # column except system_id is rebuilt, keeping the survivors' values. kept = survivors.numel() for key, default_fn in dynamics._bookkeeping_keys.items(): if key == "system_id": continue column = default_fn(refilled.num_graphs, refilled.device) if kept > 0 and key in state: carried = state[key][survivors] column[:kept] = carried.unsqueeze(-1) if carried.dim() == 1 else carried refilled[key] = column return refilled def _warn_generation_exhausted( self, config: OnPolicyConfig, target_step_count: int, finished: Batch ) -> None: """Announce that the run trains on what it has already generated. *finished* is the batch whose last trajectories graduated. Every one of them did, so its atoms and edges are the room the final backfill had to fill. """ structures = config.initial_structures if structures.exhausted: remedy = ( "Pass initial_structures=InitialStructures(dataset, recycle=True) " "to keep generating from the front of the rows this rank owns, or " "start from more structures — an unbudgeted source is propagated " "whole, so more of them lengthen the run by widening the initial " "batch rather than by backfilling it." ) else: freed = f"{int(finished.num_nodes_per_graph.sum())!r} atoms" if getattr(structures, "max_edges", None) is not None: freed += f" and {int(finished.num_edges_per_graph.sum())!r} edges" remedy = ( f"The source still holds rows, but none of them fits the {freed} " "the finished trajectories freed. Widen the source's budget, " "which sets the initial batch and with it the room a graduation " "frees." ) warnings.warn( "Every generated trajectory has finished and the initial structures " "have nothing left to start a fresh one from, so generation stopped " f"after {config.dynamics.step_count} propagator steps with " f"{len(self._replay_buffer)} frames in the replay buffer; the " f"remaining {target_step_count - self.step_count} training steps " f"draw from that buffer. {remedy}", UserWarning, stacklevel=2, ) def _resolve_replay_device( self, config: OnPolicyConfig ) -> torch.device | str | None: """Return the device the segment loop stages generated frames on. Frames reach the buffer from a host-memory sink. An unset ``replay_device`` therefore means the reference dataset's device, because the two sources are collated before the strategy moves the batch. That device is measured from a batch when no declaration settles it. A run with no reference dataset leaves the frames in host memory. The device is measured here rather than at construction, because a launcher pins the process to its device only after the datasets are built, and a ``SETUP`` hook may move a reference dataset onto this rank's device. An index-less ``replay_device`` is resolved through :func:`nvalchemi.data.resolve_device` to this process's current device. Under a launcher, that is the device the launcher pinned this rank to. Resolving it turns the caller's "this rank's GPU" into a concrete device, which the concentration check and the mixture's device comparison can work with, just as a storage records its own device. A device measured from the reference dataset is already concrete and is used as measured. Warns ----- UserWarning If, on a multi-rank world, any rank resolves an indexed accelerator other than the device it trains on. """ if config.replay_device is not None: device = resolve_device(config.replay_device) elif self.reference_dataset is None: return None else: device = dataset_device(self.reference_dataset) self._warn_concentrated_replay_device(device) return device def _warn_concentrated_replay_device(self, device: torch.device) -> None: """Report a world that stages every rank's replay frames on one accelerator. Datasets are built before a launcher pins the process. A reference dataset on an indexed device therefore emits there in every process, and the buffer follows it, because a mixed batch is collated before the strategy moves it. The world's buffers then pile onto one GPU sized for a single rank's ``replay_capacity``. Every path that resolves the device hands over an indexed one, and a rank-local dataset resolves to this rank's own device, which ``same_device`` matches against ``devices[0]`` however that entry is spelled. Each rank contributes the one bit it can see, whether it stages somewhere other than its own device, through :func:`~nvalchemi.training.distributed.all_reduce_flags`. The collective runs on every rank of a multi-rank world rather than behind a guard, so every rank joins it and reports the world's verdict. Parameters ---------- device : torch.device Device the buffer is about to stage its frames on. Warns ----- UserWarning If, on a multi-rank world, any rank stages its replay frames on an indexed accelerator other than its own device. """ world_size = get_world_size(self.distributed_manager) if world_size == 1: return elsewhere = device.type != "cpu" and not same_device(device, self.devices[0]) concentrated = all_reduce_flags(elsewhere, self.distributed_manager) if not bool(concentrated.any()): return warnings.warn( "A rank of this world stages its replay buffer and collates its " "mixture on an indexed accelerator that is not the device every rank " f"trains on; this rank stages on {device!s}. A reference dataset put " "on an indexed device before the launcher pinned the process emits " "there in every process, and the generated frames follow it, because " "a mixed batch is collated before it is moved. That GPU then holds " f"{world_size!r} replay buffers of replay_capacity frames each. Size " f"replay_capacity for {world_size!r} buffers, or open the reference " "dataset per rank: keep it in host memory, or move it onto this " "rank's device once the launcher has pinned the process.", UserWarning, stacklevel=2, ) def _rank_seed_offset(self, config: OnPolicyConfig) -> int: """Return the offset separating this rank's seeded streams from its neighbors'. Both seeded streams add a counter to their base seed, so neighboring ranks sit a whole ``rank_seed_stride`` apart rather than one. The offset is keyed on the global rank, because node-local ranks repeat across nodes. """ return get_rank(self.distributed_manager) * config.rank_seed_stride def _capture_segment( self, config: OnPolicyConfig, state: Batch, label_hook: TeacherLabelHook, buffer: ReplayBuffer, ) -> None: """Label the frame the segment ended on and drain the sink into *buffer*. The cadence rarely lands on a segment's last step, yet that frame is the most on-policy one the segment produced. The hook's forced entry point is therefore called for the step just finished. A forced label is never skipped. Labeling is idempotent per step, so the forced call costs nothing when the cadence did land on that step. """ label_hook._label_frame( state, max(config.dynamics.step_count - 1, 0), dynamics=config.dynamics, forced=True, ) if label_hook.sink is not None and len(label_hook.sink) > 0: buffer.extend(label_hook.sink.drain()) def to_spec_dict(self) -> dict[str, Any]: """Serialize declarative distillation settings to a JSON-ready dict. The bundle names its own class under ``strategy_cls``, which :meth:`from_spec_dict` builds. ``on_policy`` and ``reference_dataset`` hold a live propagator, scorer, and datasets that no spec can describe. They are therefore omitted. A rebuilt strategy runs offline unless they are passed back to :meth:`from_spec_dict`, :meth:`from_checkpoint_dict`, or :meth:`load_checkpoint`. Returns ------- dict[str, Any] JSON-ready bundle suitable for :func:`json.dumps`. Warns ----- UserWarning If ``on_policy`` is set, because the spec cannot carry it. """ spec = super().to_spec_dict() spec["strategy_cls"] = f"{type(self).__module__}.{type(self).__qualname__}" spec["teacher_signals"] = ( None if self.teacher_signals is None else sorted(self.teacher_signals) ) spec["label_missing"] = self.label_missing spec["label_dtype"] = ( None if self.label_dtype is None else str(self.label_dtype) ) if self.on_policy is not None: warnings.warn( "on_policy and reference_dataset hold live runtime objects and " "are omitted from the spec, so a strategy rebuilt from it runs " "offline over the dataloader passed to run(). Re-supply them at " "construction to keep generating on-policy.", UserWarning, stacklevel=2, ) return spec @classmethod def from_spec_dict( cls, spec: Mapping[str, Any], *, models: strategy_validation.ModelInput | None = None, hooks: Sequence[Any] | None = None, training_fn: Any = None, validation_config: ValidationConfig | None = None, on_policy: OnPolicyConfig | None = None, reference_dataset: BatchDatasetProtocol | None = None, ) -> DistillationStrategy: """Rebuild a :class:`DistillationStrategy` from ``to_spec_dict`` output. A ``strategy_cls`` naming a subclass dispatches to that class's own ``from_spec_dict`` with the spec and every runtime override, so the strategy a spec names is the one that runs; a subclass adding a runtime keyword must widen this call with it. An optional keyword is forwarded only when it is set, so a subclass overriding ``from_spec_dict`` without it still rebuilds from a plain spec. ``on_policy`` and ``reference_dataset`` must be passed with the *models* they were built around, because the propagator has to hold the very object supplied as ``models['student']``. These two and ``validation_config`` are runtime overrides: live objects that a spec cannot carry. :meth:`load_checkpoint` and :meth:`from_checkpoint_dict` forward them to this method as extra keyword arguments, and :meth:`load_checkpoint` passes its caller's *models* through the loader's own ``models`` keyword. A keyword argument here is the only way to supply them. One left ``None`` stays ``None`` on the rebuilt strategy, so a rebuild without ``on_policy`` runs offline. Parameters ---------- spec : Mapping[str, Any] A dict produced by :meth:`to_spec_dict`, optionally after a JSON round-trip. models : BaseModelMixin | dict[str, BaseModelMixin] | None, optional Runtime model override(s). Distillation models are not serialized in full, so the student and teacher are normally re-supplied here. hooks : Sequence[Any] | None, optional Runtime hooks; defaults to an empty list. training_fn : Any, optional Runtime callable or dotted-path override. validation_config : ValidationConfig | None, optional Runtime validation configuration. Specs exclude it because it carries a live loader, so a validation-only ``teacher_*`` target is resolved by passing the config here rather than assigning it afterwards, which re-runs no validator. on_policy : OnPolicyConfig | None, optional Segment loop to rebuild the run with, built around the supplied student. Default ``None`` rebuilds an offline run. reference_dataset : BatchDatasetProtocol | None, optional Reference dataset the segment loop mixes into every batch. Default ``None``. Returns ------- DistillationStrategy A freshly validated strategy of the class *spec* names, ready to :meth:`run`. Raises ------ ValueError If *spec* is missing a required key, if its ``strategy_cls`` entry is not a dotted class path string, or if that path resolves to a class that is not a :class:`DistillationStrategy` subclass. TypeError If a runtime keyword is supplied but the subclass *spec* names overrides ``from_spec_dict`` without accepting it. """ required = ("optimizer_configs", "devices", "loss_fn_spec") missing = [key for key in required if key not in spec] if missing: raise ValueError( f"from_spec_dict: spec is missing required key(s) {missing}. " f"Expected keys: {list(required)}." ) raw_strategy_cls = spec.get("strategy_cls") if raw_strategy_cls is not None: if not isinstance(raw_strategy_cls, str): raise ValueError( "from_spec_dict: 'strategy_cls' must be a dotted class path " f"string; got {type(raw_strategy_cls).__name__}." ) imported = _import_cls(raw_strategy_cls) if not issubclass(imported, cls): raise ValueError( f"from_spec_dict: {raw_strategy_cls!r} must resolve to a " f"{cls.__name__} subclass." ) if imported is not cls: return imported.from_spec_dict( spec, models=models, hooks=hooks, training_fn=training_fn, **_set_rebuild_overrides( imported, { "validation_config": validation_config, "on_policy": on_policy, "reference_dataset": reference_dataset, }, ), ) model_input = strategy_spec._models_from_spec_and_overrides( spec.get("model_specs", {}), models, single_model_input=strategy_spec._single_model_input_from_spec( spec.get("single_model_input") ), ) return cls( models=model_input, optimizer_configs=strategy_spec._optimizer_configs_from_spec( spec["optimizer_configs"] ), num_epochs=spec.get("num_epochs"), num_steps=spec.get("num_steps"), epoch_step_modifier=spec.get("epoch_step_modifier", 1.0), hooks=list(hooks) if hooks is not None else [], training_fn=strategy_spec._training_fn_from_spec(spec, training_fn), loss_fn=strategy_spec._loss_fn_from_spec(spec["loss_fn_spec"]), devices=strategy_spec._devices_from_spec(spec["devices"]), validation_config=validation_config, teacher_signals=spec.get("teacher_signals"), label_missing=spec.get("label_missing", True), label_dtype=( None if spec.get("label_dtype") is None else _dtype_deserialize(spec["label_dtype"]) ), on_policy=on_policy, reference_dataset=reference_dataset, ) @classmethod def from_checkpoint_dict( cls, spec: Mapping[str, Any], *, models: strategy_validation.ModelInput | None = None, hooks: Sequence[Any] | None = None, training_fn: Any = None, validation_config: ValidationConfig | None = None, on_policy: OnPolicyConfig | None = None, reference_dataset: BatchDatasetProtocol | None = None, **runtime_overrides: Any, ) -> DistillationStrategy: """Rebuild a strategy from checkpoint metadata, the segment loop included. It extends :meth:`~nvalchemi.training.TrainingStrategy.from_checkpoint_dict` by forwarding the runtime objects that :meth:`to_spec_dict` cannot carry to :meth:`from_spec_dict` as runtime overrides. Only the overrides that are set are forwarded, so a subclass whose ``from_spec_dict`` lacks one of them still rebuilds. Parameters ---------- spec : Mapping[str, Any] A dict produced by :meth:`to_checkpoint_dict`. models : BaseModelMixin | dict[str, BaseModelMixin] | None, optional Runtime model override(s), normally the models loaded from the checkpoint weight files. hooks : Sequence[Any] | None, optional Runtime hooks appended by the caller. training_fn : Any, optional Runtime callable or dotted-path override. validation_config : ValidationConfig | None, optional Runtime validation configuration. Specs exclude it because it carries a live loader, so a validation-only ``teacher_*`` target is resolved by passing the config here rather than assigning it afterwards, which re-runs no validator. on_policy : OnPolicyConfig | None, optional Segment loop to rebuild the run with, built around the supplied student. Default ``None`` rebuilds an offline run. reference_dataset : BatchDatasetProtocol | None, optional Reference dataset the segment loop mixes into every batch. Default ``None``. **runtime_overrides : Any Further keyword arguments a subclass's ``from_spec_dict`` accepts. Returns ------- DistillationStrategy A strategy with declarative fields and restart counters restored. """ supplied = { "validation_config": validation_config, "on_policy": on_policy, "reference_dataset": reference_dataset, } return super().from_checkpoint_dict( spec, models=models, hooks=hooks, training_fn=training_fn, **{name: value for name, value in supplied.items() if value is not None}, **runtime_overrides, ) @classmethod def load_checkpoint( cls, root_folder: Path | str, checkpoint_index: int = -1, map_location: str | torch.device | None = None, *, models: strategy_validation.ModelInput | None = None, hooks: Sequence[Any] | None = None, training_fn: Any = None, validators: Sequence[Any] | None = None, validation_config: ValidationConfig | None = None, on_policy: OnPolicyConfig | None = None, reference_dataset: BatchDatasetProtocol | None = None, **runtime_overrides: Any, ) -> DistillationStrategy: """Load a restartable checkpoint, re-supplying what the spec omits. It extends :meth:`~nvalchemi.training.TrainingStrategy.load_checkpoint` with the runtime objects a distillation spec cannot describe. They reach :meth:`from_spec_dict` as runtime overrides that the loader forwards. The segment loop is tied to the student it propagates, so *models* is re-supplied alongside *on_policy*, and the checkpoint's weights are loaded into those models. A strategy with a Boltzmann term refuses to rebuild without the loop. Parameters ---------- root_folder : Path | str Root directory containing checkpoint files. checkpoint_index : int, optional Checkpoint index to load. ``-1`` loads the latest manifest index. map_location : str | torch.device | None, optional Device override passed through to :func:`torch.load` and the restored strategy metadata. models : BaseModelMixin | dict[str, BaseModelMixin] | None, optional Models to restore the checkpoint's weights into, in place of the ones the loader builds from the saved specs: the objects the segment loop's propagator holds. Default ``None``. hooks : Sequence[Any] | None, optional Runtime hooks to attach to the restored strategy. training_fn : Any, optional Runtime training function override. validators : Sequence[Any] | None, optional Loaded-checkpoint validators forwarded to the lower-level loader. validation_config : ValidationConfig | None, optional Runtime validation configuration. Specs exclude it because it carries a live loader, so a validation-only ``teacher_*`` target is resolved by passing the config here rather than assigning it afterwards, which re-runs no validator. on_policy : OnPolicyConfig | None, optional Segment loop to restore the run with. Default ``None`` restores an offline run. reference_dataset : BatchDatasetProtocol | None, optional Reference dataset the segment loop mixes into every batch. Default ``None``. **runtime_overrides : Any Further keyword arguments a subclass's ``from_spec_dict`` accepts. Returns ------- DistillationStrategy Restored strategy with model, optimizer, scheduler, and runtime counters loaded. """ supplied = { "validation_config": validation_config, "on_policy": on_policy, "reference_dataset": reference_dataset, } return super().load_checkpoint( root_folder, checkpoint_index, map_location, models=models, hooks=hooks, training_fn=training_fn, validators=validators, **{name: value for name, value in supplied.items() if value is not None}, **runtime_overrides, )