# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Configuration of the on-policy generate-label-train segment loop."""
from __future__ import annotations
import copy
import warnings
from collections.abc import Callable, Iterator, Mapping
from contextlib import nullcontext
from typing import Annotated, Any
import torch
from jaxtyping import Bool
from pydantic import (
BaseModel,
ConfigDict,
Field,
PrivateAttr,
field_validator,
model_validator,
)
from nvalchemi.data.batch import Batch
from nvalchemi.data.datapipes.dataset import BatchDatasetProtocol
from nvalchemi.distributed.domain_parallel import DomainParallel
from nvalchemi.dynamics.base import BaseDynamics, ConvergenceHook, DynamicsStage
from nvalchemi.dynamics.sinks import DataSink, ResizableSink
from nvalchemi.hooks import DynamicsContext
from nvalchemi.training.distillation.replay import (
FIFO,
AdmissionPolicy,
EvictionPolicy,
ReplayEviction,
_batch_allocation,
_batch_size_remedy,
)
from nvalchemi.training.distillation.scoring import (
TeacherScorer,
_isolated_neighbors,
_planned_neighbor_sources,
)
from nvalchemi.training.distillation.seeding import (
InitialStructures,
InitialStructuresSource,
)
from nvalchemi.training.runtime import evaluating
__all__ = ["OnPolicyConfig", "OnPolicySettings", "ResizableSink"]
def _probe_propagator(probe: Batch, dynamics: BaseDynamics) -> Batch | None:
"""Run one ``compute()`` on *probe* and check the result against the declared keys.
:meth:`~nvalchemi.dynamics.base.BaseDynamics.check_initial_batch`
compares the declared keys with the initial structures. This function
compares them with what ``compute()`` actually does. A propagator whose
declarations no longer match its implementation is therefore rejected
here rather than on the first step of a long run. Examples are a
``__needs_keys__`` output the student does not produce, or a field that
``compute()`` reads but nothing declared. The cost is one student forward
pass at construction, which front-loads the kernel and CUDA
initialization the first step pays anyway.
The forward pass runs with the same isolation the scorer uses. The model
is held in evaluation mode, and every submodule's own ``training`` flag is
restored afterwards. ``compute()`` restores the ``requires_grad`` flags it
enables. The propagator's ``_last_outputs`` is put back. The probe batch
is the caller's own row, moved to the model's device. A graph model gets
the neighbor list its ``neighbor_config`` declares, built on the probe and
rolled back afterwards; during a run, a hook builds the propagator's list
instead. A model that plans more than one neighbor-list source is not
probed, and a warning says so. The probe builds exactly one list, and the
check must not reject a propagator the loop can run.
Parameters
----------
probe : Batch
One-row batch already checked for the declared structure fields.
dynamics : BaseDynamics
Propagator to probe.
Returns
-------
Batch | None
The probe, on the model's device, carrying the outputs ``compute()``
wrote. ``None`` when the propagator was not probed.
Raises
------
ValueError
If the model produced no output for a declared ``__needs_keys__``
entry, if ``compute()`` read a field the probe does not carry, or if
a declared ``__provides_keys__`` entry is absent afterwards.
Warns
-----
UserWarning
If the model plans more than one neighbor-list source. The
propagator's declarations then go unchecked until its first step.
"""
model = getattr(dynamics, "model", None)
if model is None:
return None
if _planned_neighbor_sources(model) > 1:
warnings.warn(
f"{type(dynamics).__name__} was not probed at construction: its "
f"model {type(model).__name__} plans more than one neighbor-list "
"source and the probe builds exactly one list, so its declared keys "
"are first checked against compute() on the first step. Pass "
"probe=False to silence this.",
UserWarning,
stacklevel=2,
)
return None
parameters = getattr(model, "parameters", None)
device = (
next((parameter.device for parameter in parameters()), None)
if callable(parameters)
else None
)
if device is not None and probe.device != device:
probe = probe.to(device)
neighbor_config = getattr(
getattr(model, "model_config", None), "neighbor_config", None
)
name = type(dynamics).__name__
last_outputs = getattr(dynamics, "_last_outputs", None)
try:
held = (
evaluating(model) if isinstance(model, torch.nn.Module) else nullcontext()
)
with held, _isolated_neighbors(probe, neighbor_config):
dynamics.compute(probe)
dynamics._validate_batch_keys(probe)
except (KeyError, AttributeError) as exc:
raise ValueError(
f"{name}.compute() read a field the initial structures do not carry "
f"({exc.args[0]!s}). Declare it in __provides_keys__ so the "
"structures are checked for it at construction, or add it to the "
"structures."
) from exc
except RuntimeError as exc:
raise ValueError(
f"{name}'s declared keys do not match what its compute() did on one "
f"initial structure: {exc}"
) from exc
finally:
dynamics._last_outputs = last_outputs
return probe
def _probe_criterion(
probe: Batch, dynamics: BaseDynamics, criterion: ConvergenceHook
) -> None:
"""Dispatch a copy of *criterion* once on *probe* and check that it migrates status.
*probe* carries the outputs of one ``compute()``. It is stamped with the
``status`` the run gives its structures and dispatched to a deep copy of
the criterion, exactly as the propagator dispatches the live one. The hook
must therefore read every key its criteria name, and the ``status`` column
must migrate to ``target_status`` exactly where
:meth:`~nvalchemi.dynamics.base.ConvergenceHook.evaluate_mask` says the
structure converged. Whether the structure converges depends on the data
and is not checked; whether the migration works is. Dispatching a copy
leaves the live criterion untouched, because the lifecycle registers and
removes that object by identity. A criterion that names a key the row does
not carry is not dispatched, because a hook may write that key during the
step and one ``compute()`` cannot show that. The check is skipped instead,
with a warning that names the key.
Parameters
----------
probe : Batch
One-row batch :func:`_probe_propagator` returned.
dynamics : BaseDynamics
Propagator the criterion will be registered on.
criterion : ConvergenceHook
Live criterion the lifecycle drives.
Raises
------
ValueError
If the criterion raised while reading the row, or if the status column
did not migrate where the criterion converged.
Warns
-----
UserWarning
If a criterion reads a key the probed row does not carry.
"""
missing = sorted({rule.key for rule in criterion.criteria if rule.key not in probe})
if missing:
warnings.warn(
f"The convergence criterion reads {missing!r}, which one compute() of "
f"{type(dynamics).__name__} on an initial structure did not produce, so "
"whether it fires cannot be checked at construction. A hook writing "
"the key during the step is fine; a key nothing writes never converges.",
UserWarning,
stacklevel=2,
)
return
hook = copy.deepcopy(criterion)
probe["status"] = torch.full(
(probe.num_graphs, 1), hook.source_status, dtype=torch.long, device=probe.device
)
try:
converged = hook.evaluate_mask(probe)
hook(
DynamicsContext(batch=probe, step_count=0, workflow=dynamics),
DynamicsStage.AFTER_STEP,
)
except (KeyError, AttributeError, RuntimeError) as exc:
raise ValueError(
"The convergence criterion failed on one initial structure carrying "
f"the propagator's outputs: {exc}"
) from exc
status = probe["status"].view(-1)
expected = torch.where(
converged,
torch.full_like(status, hook.target_status),
torch.full_like(status, hook.source_status),
)
if not torch.equal(status, expected):
raise ValueError(
"The convergence criterion fired on one initial structure but the "
f"status column did not migrate where it converged; got status "
f"{status.tolist()!r} for converged {converged.tolist()!r}, migrating "
f"{hook.source_status!r} to {hook.target_status!r}. A criterion the "
"lifecycle drives has to write batch.status itself, as ConvergenceHook "
"does."
)
def _propagator_tree(dynamics: BaseDynamics) -> Iterator[BaseDynamics]:
"""Yield *dynamics* and every propagator it composes, each exactly once.
A :class:`~nvalchemi.dynamics.FusedStage` keeps its integrators in
``sub_stages``, and a pipeline keeps its stages in ``stages``. A check that
reads only the root therefore misses the propagator that actually runs.
Nodes are compared by identity, because one integrator reached through two
sub-stages is a single propagator.
Parameters
----------
dynamics : BaseDynamics
Propagator at the root of the composition.
Yields
------
BaseDynamics
Every propagator in the tree, the root first.
"""
seen: list[BaseDynamics] = []
pending: list[BaseDynamics] = [dynamics]
while pending:
node = pending.pop()
if any(node is visited for visited in seen):
continue
seen.append(node)
yield node
pending.extend(sub for _, sub in getattr(node, "sub_stages", ()))
stages = getattr(node, "stages", ())
pending.extend(stages.values() if isinstance(stages, Mapping) else stages)
def _status_migrators(dynamics: BaseDynamics) -> list[ConvergenceHook]:
"""Return every status migrator on *dynamics* and its sub-stages.
A status migrator is a :class:`~nvalchemi.dynamics.base.ConvergenceHook`
with both ``source_status`` and ``target_status`` set. Every place a
propagator can hold one is searched. That covers its registered hooks (on
a :class:`~nvalchemi.dynamics.FusedStage`, the hooks registered at the
fused level) and its ``convergence_hook``. It also covers the same two
places on every sub-stage, where a fused stage puts the migrators it
builds itself: one on every sub-stage except the last, and one on the
last whenever it declares a ``convergence_hook``.
Parameters
----------
dynamics : BaseDynamics
Propagator to walk.
Returns
-------
list[ConvergenceHook]
The migrators, in the order they were found.
"""
return [
hook
for propagator in _propagator_tree(dynamics)
for hook in (*propagator.hooks, propagator.convergence_hook)
if isinstance(hook, ConvergenceHook)
and hook.source_status is not None
and hook.target_status is not None
]
def _competing_migrators(
dynamics: BaseDynamics, criterion: ConvergenceHook
) -> list[ConvergenceHook]:
"""Return the status migrators already on *dynamics* that are not *criterion*.
The lifecycle is about to install *criterion* as the propagator's
``convergence_hook`` and as a registered hook, so *criterion* itself is
not a competitor wherever the walk finds it.
Parameters
----------
dynamics : BaseDynamics
Propagator the lifecycle is being installed on.
criterion : ConvergenceHook
The lifecycle's own criterion.
Returns
-------
list[ConvergenceHook]
The competing criteria, in the order they were found.
"""
return [hook for hook in _status_migrators(dynamics) if hook is not criterion]
def _check_sole_migrator(dynamics: BaseDynamics, criterion: ConvergenceHook) -> None:
"""Reject *dynamics* unless *criterion* would be its only status migrator.
The config runs this check when it is built, and the lifecycle runs it
again when the run starts, because a hook can be registered on the
propagator in between.
Raises
------
ValueError
If *dynamics* carries a status-migrating criterion other than
*criterion*.
"""
competing = _competing_migrators(dynamics, criterion)
if not competing:
return
migrations = [(hook.source_status, hook.target_status) for hook in competing]
raise ValueError(
"The relaxation lifecycle owns graduation for this run, so the "
"propagator must carry no other status-migrating ConvergenceHook; "
f"got {migrations!r} beside the configured "
f"({criterion.source_status!r}, {criterion.target_status!r}). "
"Remove it, or drop fmax or convergence_hook and let the propagator "
"manage its own lifecycle. A FusedStage builds one for every "
"sub-stage except the last, and for the last one when it declares a "
"convergence_hook."
)
def _check_structure_status(state: Batch, criterion: ConvergenceHook) -> None:
"""Reject a criterion that migrates off a status no initial structure holds.
:meth:`~nvalchemi.dynamics.base.ConvergenceHook.__call__` migrates only the
graphs whose status equals its ``source_status``. A criterion whose
``source_status`` no initial structure holds leaves the convergence path
inert: no structure ever converges, so none freezes or graduates through
it, and generation never runs out of trajectories through it, so the
exhaustion warning never fires. The divergence path ignores
``source_status``, so a diverged structure would still freeze, graduate,
and be counted in the boundary warning.
Parameters
----------
state : Batch
Initial batch, already stamped with the run's own bookkeeping.
criterion : ConvergenceHook
Criterion driving the trajectory lifecycle.
Raises
------
ValueError
If *state* carries no ``status`` column, or if no graph of it carries
the criterion's ``source_status``.
"""
if "status" not in state:
raise ValueError(
"The initial batch carries no status column, so nothing could migrate "
f"off source_status={criterion.source_status!r}. A relaxation "
"lifecycle graduates structures on that column. An "
"InitialStructuresSource driving a lifecycle stamps status zeros and "
"system_ids on the batch its initial_batch returns, as "
"InitialStructures does."
)
statuses = sorted({int(value) for value in state["status"].view(-1).tolist()})
if criterion.source_status in statuses:
return
raise ValueError(
"A converged graph migrates off the status its initial structure "
f"carries; got source_status={criterion.source_status!r} against "
f"initial statuses {statuses!r}. The run stamps that status itself "
"rather than reading it from the structures, so nothing would ever "
"freeze or graduate. Pass source_status=0, or pass the threshold as "
"fmax instead."
)
[docs]
class OnPolicySettings(BaseModel):
"""Declarative settings of one on-policy distillation segment loop.
Every field is a JSON scalar, so the settings validate without a
propagator, a teacher, or a store. A recipe with invalid settings is
therefore rejected before any teacher is loaded. :class:`OnPolicyConfig`
inherits these settings and adds the live objects the loop drives. That
class and the strategy check whether those objects work with the loop.
Parameters
----------
replay_ratio : float
Fraction of every training batch drawn from the replay buffer. The
reference dataset supplies the rest.
training_steps_per_segment : int
Optimizer steps taken per segment, one per training batch.
batch_size : int, optional
Samples per training batch, counting both sources of the mixture.
Default ``8``.
generation_steps : int, optional
Propagator steps generated per segment. Default ``100``.
label_frequency : int, optional
Propagator steps between teacher labelings. Each segment's last frame
is labeled in addition. Default ``100``.
replay_capacity : int | None, optional
Frame capacity of the replay buffer. Default ``None`` (unbounded). A
Boltzmann objective needs a bound; see the :class:`OnPolicyConfig`
Notes.
replay_eviction : {"fifo"}, optional
Eviction policy of the replay buffer, as a name a recipe can store. A
policy instance is passed to :class:`OnPolicyConfig` instead. Default
``"fifo"``.
replay_device : str | None, optional
Device the replay buffer keeps frames on. An index-less ``cuda`` means
this rank's current CUDA device. Default ``None`` uses the
device the reference dataset emits its batches on, or host memory when
there is no reference dataset.
seed : int, optional
Base seed of every segment's mixture sampler. Default ``0``.
rank_seed_stride : int, optional
Seed offset between neighboring ranks on a multi-rank launch. Rank
``r`` adds ``r * rank_seed_stride`` to its seeds. Default
``1_000_003``.
require_wrapped_student : bool, optional
Whether a multi-rank run refuses to start unless the ``SETUP`` stage
replaced the student with a wrapper that owns it, as a ``DDPHook``
does. Default ``True``.
samples_equilibrium : bool | None, optional
Whether the propagator samples an equilibrium ensemble, the ensemble a
distribution-matching objective is defined on. Default ``None`` infers
the answer from the propagator: a stage whose class declares
:attr:`~nvalchemi.dynamics.BaseDynamics.samples_equilibrium` false, as
the relaxation optimizers do, or a convergence criterion counts as not
sampling one. ``True`` or ``False`` overrides that inference for the
objective's guard.
fmax : float | None, optional
Max force norm below which a generated trajectory counts as finished.
Setting it turns on the trajectory lifecycle of a relaxation run, in
which converged structures leave the batch and fresh initial
structures replace them. Default ``None`` ends no trajectory, which
suits a molecular-dynamics run.
max_trajectory_steps : int | None, optional
Propagator steps a generated trajectory lives for. A trajectory that
reaches this age is frozen on that step, stored once, and retired at
the segment boundary like a converged one, so a fresh initial
structure takes its place. It turns on the trajectory lifecycle on
its own, which is how a thermostatted run rotates through its initial
structures instead of propagating the first batch forever. A multiple
of ``generation_steps`` retires on segment boundaries, so no frozen
graph idles inside a segment. Default ``None`` lets a trajectory live
for the whole run.
quarantine_diverged : bool, optional
Sanitize a diverged graph on the step it is flagged: its velocities
and forces are zeroed and its positions made finite, so the frozen
graph presents a finite geometry to the model on every remaining step
of the segment. The propagator still evaluates the whole batch, frozen
graphs included, and a non-finite or runaway geometry can break a
model's neighbor-list construction. The sanitized frame is never
captured. Default ``False`` leaves the flagged state as it is.
weight_sync_frequency : int, optional
Segments between weight syncs to the propagator. Default ``1``, the
only accepted value while the propagator shares the student module.
probe : bool, optional
Whether :class:`OnPolicyConfig` runs the propagator's ``compute()`` on
one initial structure at construction to check it against its
declared keys. ``False`` defers any mismatch to the first step.
Default ``True``.
Raises
------
ValueError
If a count or the threshold is not positive, if ``replay_ratio`` falls
outside ``[0, 1]`` or is exactly ``0``, if rounding the ratio against
the batch size gives one mixture source no sample in any batch, or if
``weight_sync_frequency`` is not ``1``.
Examples
--------
>>> from nvalchemi.training.distillation import OnPolicySettings
>>> settings = OnPolicySettings(replay_ratio=0.25, training_steps_per_segment=32)
>>> settings.batch_size
8
Notes
-----
``label_frequency`` is the throughput setting. It counts against the
propagator's cumulative ``step_count``, so the labeling cadence does not
restart at a segment boundary. Each segment also labels the frame it ends
on, and the cadence dispatch adjacent to that forced label is skipped.
With ``generation_steps`` a multiple of ``label_frequency``, each
trajectory is therefore labeled ``generation_steps // label_frequency``
times per segment, which is once per segment only when the two are equal.
The first segment pays one more, for the cadence dispatch at step ``0``.
``training_steps_per_segment`` counts training batches. It equals the
number of optimizer steps only while every batch takes one step.
``fmax`` is compared against the student's forces, which are the forces
the propagator follows. It therefore measures the convergence of the
relaxation itself.
Size ``replay_capacity`` as a multiple of the trajectory count. Otherwise
FIFO eviction removes only part of one labeled step's frames, which
over-represents the structures at the back of the batch. Space the
``seed`` of replicate runs by at least
``num_steps // training_steps_per_segment``, because the sampler adds the
segment index to it. See :ref:`training-distillation-api`.
On a multi-rank launch, each rank adds ``rank * rank_seed_stride`` to
``seed`` and, through
:meth:`~nvalchemi.dynamics.BaseDynamics.seed_offset`, to every
``random_seed`` that ``dynamics`` and its sub-stages expose. Ranks
therefore draw from the reference dataset independently and apply
different thermostat noise to the structures they were dealt. Both
streams add a step counter to their seed, so the stride has to stay above
every counter the run reaches; the default, ``1_000_003``, does so for a
run whose counters stay below it. Set a different stride when a replicate
launch's seeds would land on another rank's stride. A stage that holds
randomness the offset cannot move, such as a :class:`torch.Generator`
with no integer ``random_seed``, is named in a warning, and the caller
must give it a rank-distinct seed.
A multi-rank run also checks that the ``SETUP`` stage replaced the student
with a gradient-synchronizing wrapper. A wrapper that works in place, such
as FSDP2's ``fully_shard`` or hook-based synchronization, leaves the
student object unchanged and fails that check.
``require_wrapped_student=False`` waives the check for such a wrapper, and
keeping the ranks' students in step then becomes the caller's
responsibility.
"""
replay_ratio: Annotated[
float,
Field(
ge=0.0,
le=1.0,
description=(
"Fraction of every training batch drawn from the replay buffer; "
"the rest comes from the reference dataset."
),
),
]
training_steps_per_segment: Annotated[
int,
Field(
gt=0,
description=(
"Training batches drawn from each segment's mixture, one "
"optimizer step each unless an update hook vetoes the step."
),
),
]
batch_size: Annotated[
int,
Field(
default=8,
gt=0,
description=(
"Samples per training batch, split between the reference "
"dataset and the replay buffer at replay_ratio."
),
),
] = 8
generation_steps: Annotated[
int,
Field(
default=100,
gt=0,
description="Propagator steps generated per segment.",
),
] = 100
label_frequency: Annotated[
int,
Field(
default=100,
gt=0,
description=(
"Propagator steps between teacher labelings, on top of the "
"segment's own last frame. Larger values trade label density "
"for generation throughput."
),
),
] = 100
replay_capacity: Annotated[
int | None,
Field(
default=None,
gt=0,
description="Frames the replay buffer keeps; None leaves it unbounded.",
),
] = None
replay_eviction: Annotated[
ReplayEviction,
Field(
default="fifo",
description=(
"Policy retiring frames from a full replay buffer, named as a "
"recipe spells it; 'fifo' drops the oldest frames first."
),
),
] = "fifo"
replay_device: Annotated[
str | None,
Field(
default=None,
description=(
"Device the replay buffer keeps frames on, named as a string. "
"None uses the device the reference dataset emits its batches "
"on, so the mixture collates on one device, or host memory when "
"the run has no reference dataset. An index-less 'cuda' names "
"the device this rank has made current, which under a launcher "
"is the one it pinned this rank to."
),
),
] = None
seed: Annotated[
int,
Field(
default=0,
ge=0,
description=(
"Base seed of every segment's mixture sampler, combined with the "
"segment index so consecutive segments draw different reference "
"samples and replicate runs can be made independent."
),
),
] = 0
rank_seed_stride: Annotated[
int,
Field(
default=1_000_003,
gt=0,
description=(
"Seed-space distance between neighboring ranks: rank r moves "
"seed, and every random_seed the propagator exposes, by "
"r * rank_seed_stride. Both streams add a step counter to the "
"base seed, so keep it above the run's step count. Set a "
"different stride when a replicate launch's seeds would land "
"on another rank's stride."
),
),
] = 1_000_003
require_wrapped_student: Annotated[
bool,
Field(
default=True,
description=(
"Whether a multi-rank run refuses to start unless the SETUP "
"stage replaced models['student'] with a wrapper owning it, "
"the way a DDPHook does. False skips that check with a warning "
"for wrappers working in place (FSDP2 fully_shard, hook-based "
"gradient synchronization) and leaves keeping the ranks' "
"students in step to the caller."
),
),
] = True
fmax: Annotated[
float | None,
Field(
default=None,
gt=0.0,
description=(
"Max force norm below which a generated trajectory counts as "
"finished, which is what turns a relaxation run into a "
"lifecycle. None runs no trajectory lifecycle: nothing graduates "
"and nothing is backfilled, which is what a molecular-dynamics "
"run wants."
),
),
] = None
max_trajectory_steps: Annotated[
int | None,
Field(
default=None,
gt=0,
description=(
"Propagator steps a generated trajectory lives for before it is "
"retired and a fresh initial structure takes its place. Turns on "
"the trajectory lifecycle on its own. None lets every "
"trajectory live for the whole run."
),
),
] = None
quarantine_diverged: Annotated[
bool,
Field(
default=False,
description=(
"Zero the velocities and forces of a diverged graph and make its "
"positions finite on the step it is flagged, so the frozen graph "
"stays a finite input to the model until the segment boundary "
"retires it. False leaves the flagged state untouched."
),
),
] = False
weight_sync_frequency: Annotated[
int,
Field(
default=1,
gt=0,
description=(
"Segments between weight syncs to the propagator. Reserved: "
"must be 1 while the propagator shares the student module."
),
),
] = 1
probe: Annotated[
bool,
Field(
default=True,
description=(
"Whether the propagator's compute() is run on one initial "
"structure at construction to check its declared keys against "
"what it does. False skips that forward and defers a mismatch "
"to the first step."
),
),
] = True
samples_equilibrium: Annotated[
bool | None,
Field(
default=None,
description=(
"Whether the propagator samples an equilibrium ensemble. None "
"infers it: a stage declaring samples_equilibrium=False on its "
"class, as the relaxation optimizers do, or a convergence "
"criterion is treated as not sampling equilibrium. True or "
"False overrides that inference for a distribution-matching "
"objective's guard."
),
),
] = None
model_config = ConfigDict(extra="forbid")
@field_validator("replay_device", mode="before")
@classmethod
def _name_replay_device(cls, value: Any) -> Any:
"""Accept a ``torch.device`` and store it as the string every reader expects."""
return str(value) if isinstance(value, torch.device) else value
@model_validator(mode="after")
def _validate_weight_sync(self) -> OnPolicySettings:
"""Reject any ``weight_sync_frequency`` other than the reserved value 1."""
if self.weight_sync_frequency != 1:
raise ValueError(
"weight_sync_frequency must be 1: the propagator holds the same "
"student module the trainer updates, so an eager run is never out "
f"of sync; got {self.weight_sync_frequency!r}. Larger values are "
"reserved for the compiled and asynchronous teacher paths."
)
return self
@model_validator(mode="after")
def _validate_mixture(self) -> OnPolicySettings:
"""Reject a mixture that no batch can actually be drawn from."""
if self.replay_ratio == 0.0:
raise ValueError(
"replay_ratio=0 trains on reference data only, which is "
"offline distillation paying for generation it never uses; "
"drop on_policy and call run() with a loader over the labeled "
"dataset instead."
)
reference_samples, replay_samples = _batch_allocation(
self.replay_ratio, self.batch_size
)
if self.replay_ratio >= 1.0 or min(reference_samples, replay_samples) > 0:
return self
raise ValueError(
"The mixture is drawn as whole samples, so replay_ratio and "
"batch_size must together give each source at least one sample per "
f"batch; got replay_ratio={self.replay_ratio!r} with "
f"batch_size={self.batch_size!r}, which puts {reference_samples} "
f"reference and {replay_samples} generated samples in every batch "
"and leaves one source out of training entirely. To fix it, "
f"{_batch_size_remedy(self.replay_ratio)}."
)
[docs]
class OnPolicyConfig(OnPolicySettings):
"""Settings and live objects of one on-policy distillation segment loop.
Each *segment* has two phases. The *generation* phase runs the student's
own propagator for ``generation_steps`` steps and labels frames with the
teacher as it goes. The *training* phase then takes
``training_steps_per_segment`` optimizer steps. Its batches are a
*mixture* of the reference dataset and the replay buffer, split at
``replay_ratio``. The propagator holds the module the trainer updates, so
each segment generates from a fresher policy than the last. The scalar
settings come from :class:`OnPolicySettings`, which this class inherits so
that a recipe stays flat. :attr:`settings` returns a detached copy of them
for a pre-flight check or a restart bundle.
The propagator can be any :class:`~nvalchemi.dynamics.base.BaseDynamics`.
A relaxation optimizer such as :class:`~nvalchemi.dynamics.optimizers.FIRE2`
drives the loop exactly as a thermostat does. The initial structures must
carry every field the propagator updates in place through
``__provides_keys__``. For every shipped propagator these include
``velocities``, and the variable-cell ones also need a ``cell``. The model
outputs named by ``__needs_keys__`` are computed before the first step, so
they need not be present. Construction checks one row, so a missing field
is a construction error rather than a failure on the first step. The
propagator's ``compute()`` then runs once on that row. This rejects
declarations that no longer match the implementation, such as a
``__needs_keys__`` output the student never produces or a field
``compute()`` reads that nothing declared. The cost is one student forward
pass at construction. A graph model is probed with the neighbor list its
``neighbor_config`` declares, built on the row and rolled back afterwards.
A model that plans more than one neighbor-list source is not probed.
A relaxation propagator needs a *trajectory lifecycle*. Relaxations
converge, and a converged structure that keeps being propagated fills the
replay buffer with near-duplicates of a frame it already holds. ``fmax``
turns the lifecycle on. A converged structure freezes and is stored once,
as the minimum it reached. At the segment boundary it *graduates*, that
is, it leaves the batch, and the initial structures *backfill* a fresh
structure in its place for as long as the source holds rows. An
:class:`~nvalchemi.training.distillation.InitialStructures` built with
``recycle=True`` wraps to its first row instead of letting the batch
narrow. Generation ends when the last trajectory finishes, and the
remaining training steps draw on the frames already in the buffer.
A thermostatted propagator needs the same lifecycle for a different
reason: its trajectories never end, so the initial batch is the only set
of structures the run ever propagates. ``max_trajectory_steps`` retires a
trajectory after that many propagator steps, stored once like a converged
one, and backfills its place from the initial structures, so the run
rotates through the whole source. It works alone or beside ``fmax``.
Parameters
----------
dynamics : BaseDynamics
Propagator that generates on-policy frames. It holds the student
module.
teacher_scorer : TeacherScorer
Scorer that labels generated frames. A custom scorer that declares
``label_fields`` lets the fields it writes be known before the run.
initial_structures : InitialStructuresSource
Structures the generated trajectories start from. They are served
from one position that backfills and restarts share. A multi-rank
launch deals them out strided, every ``world_size``-th structure to
each rank. Pass an
:class:`~nvalchemi.training.distillation.InitialStructures`, any other
object that implements the protocol, or a bare dataset, which is
wrapped in an ``InitialStructures``.
capture_sink : DataSink | None, optional
Sink that stages each segment's labeled frames until the segment
boundary drains them into the replay buffer. A
:class:`~nvalchemi.dynamics.sinks.GPUBuffer` keeps the staged frames on
the generation device and avoids a device-to-host copy per frame.
Default ``None`` builds a host-memory sink per segment.
replay_eviction : {"fifo"} | EvictionPolicy, optional
Eviction policy of the replay buffer: the name ``"fifo"`` or a live
:class:`~nvalchemi.training.distillation.EvictionPolicy` instance.
Default ``"fifo"``.
replay_admission : AdmissionPolicy | None, optional
Predicate that selects which captured frames each segment admits into
the replay buffer. Default ``None`` admits every captured frame.
convergence_hook : ConvergenceHook | None, optional
Live criterion that decides when a generated trajectory is finished,
used instead of the ``fmax`` threshold. Default ``None``.
divergence : Callable[[Batch], Bool[torch.Tensor, "G"]] | None, optional
Predicate over the live frame that returns one boolean per graph,
``True`` where the trajectory diverged. The lifecycle freezes a flagged
graph on the step it is flagged and keeps it out of both capture
routes. At the segment boundary it retires the graph and backfills its
place. Default ``None`` uses
:func:`~nvalchemi.training.distillation.nonfinite_divergence`.
Raises
------
ValueError
If a setting is out of range, if both ``fmax`` and
``convergence_hook`` are set, if ``convergence_hook`` cannot manage
the lifecycle, if ``initial_structures`` recycles while neither a
criterion nor ``max_trajectory_steps`` is set, if a lifecycle is paired with a
:class:`~nvalchemi.distributed.DomainParallel` propagator or a
multi-sub-stage :class:`~nvalchemi.dynamics.FusedStage`, if
``initial_structures`` is neither a source nor a dataset, if the
initial structures lack a field the propagator needs for its first
step, or if the propagator's ``compute()`` on one row contradicts its
declared keys.
Examples
--------
>>> from nvalchemi.training.distillation import ( # doctest: +SKIP
... InProcessTeacherScorer,
... OnPolicyConfig,
... InitialStructures,
... )
>>> config = OnPolicyConfig( # doctest: +SKIP
... dynamics=NVTLangevin(student, dt=0.5, temperature=300.0),
... teacher_scorer=InProcessTeacherScorer(teacher, ["energy", "forces"]),
... initial_structures=InitialStructures(dataset),
... replay_ratio=0.25,
... training_steps_per_segment=32,
... batch_size=16,
... generation_steps=50,
... label_frequency=10,
... replay_capacity=8192,
... )
The same loop over relaxation paths. Each structure graduates once its max
force norm falls below ``0.05``, and the next initial structure takes its
place:
>>> config = OnPolicyConfig( # doctest: +SKIP
... dynamics=FIRE2(student, dt=0.05),
... teacher_scorer=InProcessTeacherScorer(teacher, ["energy", "forces"]),
... initial_structures=InitialStructures(dataset, recycle=True),
... fmax=0.05,
... replay_ratio=0.25,
... training_steps_per_segment=32,
... batch_size=16,
... generation_steps=50,
... label_frequency=10,
... )
Notes
-----
Any :class:`~nvalchemi.training.distillation.TeacherScorer` may drive
generation. A custom scorer can declare ``label_fields``. The declaration
lets :class:`~nvalchemi.training.distillation.DistillationStrategy` check
the generated fields against ``reference_dataset`` at construction. It
also keeps :class:`~nvalchemi.training.distillation.TeacherLabelHook` from
re-scoring a frame that is dispatched again. A custom ``teacher_*`` field
the scorer writes is an ordinary loss target, so the reference dataset and
any validation data must carry it too.
The loop sizes ``capture_sink``. A segment captures at most one frame per
trajectory per labeled step, including the forced last frame. The sink
must therefore hold ``(generation_steps + 1)`` frames per trajectory in
the propagated batch. A configured sink with less capacity is grown
through ``resize(capacity)`` when it satisfies
:class:`~nvalchemi.dynamics.ResizableSink`, and rejected otherwise. The
sink must also be empty when a segment starts, because everything it holds
is drained into the replay buffer as generated frames.
``capture_sink`` is runtime-only, like ``dynamics`` and
``teacher_scorer``, so no recipe names it. A recipe does not name a policy
instance either. :attr:`settings` records a custom ``replay_eviction`` as
``"fifo"`` with a warning, and a config rebuilt from those settings evicts
FIFO until the policy is supplied again.
``fmax`` stays a plain number that a recipe can hold.
:attr:`convergence_criterion` is the live criterion the lifecycle drives.
For an ``fmax`` threshold, it is built once with
:meth:`~nvalchemi.dynamics.base.ConvergenceHook.from_fmax`, migrating
status ``0`` to the propagator's ``exit_status``. The same object is
returned on every read, because the lifecycle registers that one object
and removes it by identity. A criterion that has to be a live hook goes to
``convergence_hook`` instead. Setting both ``fmax`` and
``convergence_hook`` is refused.
A ``convergence_hook`` must migrate status on every step, from the status
``0`` the run stamps its structures with. A hook that only reports
convergence would freeze and graduate nothing. A hook that skips steps
graduates a structure late, so both capture routes would store it. The
construction probe also dispatches a copy of the criterion to the probed
row. A criterion that raises on the propagator's outputs, or whose firing
leaves ``status`` unchanged, is therefore refused here. A criterion that
reads a key no ``compute()`` produces is not dispatched, because a hook
may write that key during the step; a warning names the key instead.
``probe=False`` skips this dispatch along with the forward pass it reads.
``divergence`` ends a trajectory before it converges. It has the same
shape as :class:`~nvalchemi.training.distillation.AdmissionPolicy`: it
takes the live frame and returns one boolean per graph. The default flags
a graph whose positions or forces are no longer finite. Use a custom
predicate for a student whose forces explode to finite but unphysical
values, or for a criterion on the energy. Like ``replay_admission``, the
predicate is runtime-only. A predicate that returns anything but one
boolean per graph is refused on its first dispatch, and the error names
the shape it returned.
For the duration of the loop, the criterion also becomes the propagator's
convergence detector. It must be the only hook that migrates status, so a
propagator that carries a second migrating
:class:`~nvalchemi.dynamics.base.ConvergenceHook` is refused. A
:class:`~nvalchemi.dynamics.FusedStage` builds such a hook for every
sub-stage except the last, and for the last one whenever it declares a
``convergence_hook``. Only a single-sub-stage fused stage without a
criterion of its own is therefore accepted. A
:class:`~nvalchemi.distributed.DomainParallel` propagator is refused with
a criterion as well: its step dispatches no ``ON_GRADUATE``, the stage
the converged route captures at, so its minima would never be stored. A
migrator already on the propagator is refused when the config is built,
and one registered afterwards is refused when the run starts. See
:ref:`training-distillation-api` for the capture routes and the backfill.
Distribution-matching objectives are defined on equilibrium ensembles, and
a relaxation path is not one. A
:class:`~nvalchemi.training.distillation.BoltzmannMatchingLoss` is
therefore refused at construction beside a relaxation propagator or any
convergence criterion, unless ``samples_equilibrium=True`` declares that
the propagator does sample an equilibrium ensemble. ``False`` refuses the
term whatever the propagator is. The term should also have a bounded
``replay_capacity``. Energy, force, and atomic-energy matching are
pointwise, so they distill a relaxation path exactly as they distill a
trajectory.
"""
dynamics: Annotated[
BaseDynamics,
Field(
description=(
"Propagator generating on-policy frames from the student. Any "
"BaseDynamics: an integrator for trajectories, an optimizer for "
"relaxation paths."
)
),
]
teacher_scorer: Annotated[
TeacherScorer,
Field(
description=(
"Scorer producing the teacher signals for generated frames. A "
"label_fields declaration on a custom one lets the strategy "
"check its fields against reference_dataset up front and makes a "
"teacher_* field of its own usable as a loss target."
)
),
]
initial_structures: Annotated[
InitialStructuresSource,
Field(
description=(
"Structures the generated trajectories start from: any "
"InitialStructuresSource, of which InitialStructures is the "
"reference. The initial batch, the backfill, and a restart all "
"read from its one position. A bare dataset is wrapped in an "
"unbudgeted InitialStructures."
)
),
]
replay_eviction: Annotated[
ReplayEviction | EvictionPolicy,
Field(
default="fifo",
description=(
"Policy retiring frames from a full replay buffer: 'fifo', or a "
"live EvictionPolicy instance, which is runtime-only and "
"recorded as 'fifo' in the declarative settings."
),
),
] = "fifo"
replay_admission: Annotated[
AdmissionPolicy | None,
Field(
default=None,
description=(
"Predicate over a batch of captured frames returning one boolean "
"per graph; frames it refuses never enter the replay buffer. "
"Runtime-only: no recipe names it."
),
),
] = None
capture_sink: Annotated[
DataSink | None,
Field(
default=None,
description=(
"Runtime-only sink the labeling hook stages each segment's "
"labeled frames in until the segment boundary drains them into "
"the replay buffer; None builds a host-memory sink per segment, "
"and a GPUBuffer keeps the staging on the generation device. "
"The loop sizes it to (generation_steps + 1) frames per "
"trajectory, growing a ResizableSink through resize(capacity) "
"and refusing a smaller sink otherwise."
),
),
] = None
convergence_hook: Annotated[
ConvergenceHook | None,
Field(
default=None,
description=(
"Live criterion deciding when a generated trajectory is "
"finished, in place of the fmax threshold. No recipe "
"describes it, so it is runtime-only."
),
),
] = None
divergence: Annotated[
Callable[[Batch], Bool[torch.Tensor, "G"]] | None,
Field(
default=None,
description=(
"Predicate over the live frame returning one boolean per graph, "
"set where the trajectory diverged; the lifecycle moves the "
"verdict onto the batch's device, then freezes and retires "
"those graphs uncaptured. None flags non-finite positions or "
"forces. Runtime-only: no recipe names it."
),
),
] = None
model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
_probed: bool = PrivateAttr(default=False)
_convergence_criterion: ConvergenceHook | None = PrivateAttr(default=None)
@property
def settings(self) -> OnPolicySettings:
"""Detached copy of the declarative settings, for a recipe or a restart bundle.
Returns
-------
OnPolicySettings
The scalar settings of this config, validated on their own. The
copy holds no reference to the live objects.
Warns
-----
UserWarning
If ``replay_eviction`` is a policy instance other than
:class:`~nvalchemi.training.distillation.FIFO`. The copy records
it as ``"fifo"``.
"""
values = {name: getattr(self, name) for name in OnPolicySettings.model_fields}
eviction = self.replay_eviction
if not isinstance(eviction, str):
if not isinstance(eviction, FIFO):
warnings.warn(
f"replay_eviction is a {type(eviction).__name__} instance, "
"which no recipe or restart bundle can name, so the "
"declarative settings record 'fifo'; a config rebuilt from "
"them evicts FIFO until the policy is re-supplied at "
"construction.",
UserWarning,
stacklevel=2,
)
values["replay_eviction"] = "fifo"
return OnPolicySettings.model_validate(values)
@property
def convergence_criterion(self) -> ConvergenceHook | None:
"""Return the criterion the trajectory lifecycle drives, or ``None``.
A ``convergence_hook`` is returned as is. An ``fmax`` threshold is
turned into a criterion on first read, migrating status ``0`` to the
propagator's ``exit_status``. The same object is returned for the life
of the config, because the lifecycle registers it on the propagator
and removes it again by identity.
Returns
-------
ConvergenceHook | None
The live criterion, or ``None`` for a run managing no lifecycle.
"""
if self.convergence_hook is not None:
return self.convergence_hook
if self.fmax is None:
return None
if self._convergence_criterion is None:
self._convergence_criterion = ConvergenceHook.from_fmax(
float(self.fmax),
source_status=0,
target_status=self.dynamics.exit_status,
)
return self._convergence_criterion
@property
def manages_lifecycle(self) -> bool:
"""Whether the segment loop retires and backfills trajectories.
Returns
-------
bool
``True`` when a convergence criterion (``fmax`` or
``convergence_hook``) or a ``max_trajectory_steps`` lifetime is
set; ``False`` for a run that propagates its initial batch for
the whole run.
"""
return self.convergence_criterion is not None or (
self.max_trajectory_steps is not None
)
@model_validator(mode="before")
@classmethod
def _coerce_initial_structures(cls, data: Any) -> Any:
"""Pass a source through, wrap a bare dataset, and refuse anything else."""
if not isinstance(data, dict):
return data
data = dict(data)
structures = data.get("initial_structures")
if structures is None or isinstance(structures, InitialStructuresSource):
return data
if isinstance(structures, BatchDatasetProtocol):
data["initial_structures"] = InitialStructures(structures)
return data
raise ValueError(
"OnPolicyConfig.initial_structures must be an InitialStructuresSource "
"(an object with probe, initial_batch, shard, exhausted, draw, "
"state_dict, and load_state_dict, as InitialStructures implements "
"them) or a BatchDatasetProtocol dataset to wrap in one; got "
f"{type(structures).__name__!r}. Pass an InitialStructures, another "
"source, or a dataset."
)
@model_validator(mode="after")
def _validate_convergence_hook(self) -> OnPolicyConfig:
"""Validate a ``convergence_hook``; an ``fmax`` threshold needs no checks."""
if self.convergence_hook is None:
return self
if self.fmax is not None:
raise ValueError(
"Set fmax or convergence_hook, not both; got "
f"fmax={self.fmax!r}, convergence_hook={self.convergence_hook!r}. "
"Drop the threshold to keep the hook, or drop the hook to keep a "
"config a recipe can describe."
)
exit_status = self.dynamics.exit_status
migrates = (
self.convergence_hook.source_status is not None
and self.convergence_hook.target_status is not None
)
if not migrates:
raise ValueError(
"The convergence hook of a relaxation loop has to migrate "
f"status; got source_status={self.convergence_hook.source_status!r} "
f"and target_status={self.convergence_hook.target_status!r}. A "
"converged graph freezes in the propagator's step on its status. "
"It graduates out of the batch on that status too. Pass "
f"source_status=0 with target_status={exit_status!r}, or pass "
"the threshold as fmax instead."
)
if self.convergence_hook.target_status < exit_status:
raise ValueError(
"Converged graphs must migrate to at least the propagator's "
"exit status, which is what graduates them out of the active "
f"batch; got target_status="
f"{self.convergence_hook.target_status!r} against "
f"dynamics.exit_status={exit_status!r}."
)
if self.convergence_hook.frequency != 1:
raise ValueError(
"The convergence hook of a relaxation loop has to run on every "
f"step; got frequency={self.convergence_hook.frequency!r}. A "
"structure is captured on the step it converges. It has to be "
"frozen and left out of the path capture on that same step. A "
"hook that skips steps would store it by both routes and keep "
"propagating it until the next firing. Pass frequency=1, or pass "
"the threshold as fmax instead."
)
return self
@model_validator(mode="after")
def _validate_lifecycle_shape(self) -> OnPolicyConfig:
"""Reject a lifecycle the structures or the propagator cannot carry."""
managed = self.manages_lifecycle
if getattr(self.initial_structures, "recycle", False) and not managed:
raise ValueError(
"The initial-structures source sets recycle=True; got "
f"fmax={self.fmax!r}, "
f"convergence_hook={self.convergence_hook!r}, and "
f"max_trajectory_steps={self.max_trajectory_steps!r}. Recycling "
"wraps the backfill to the front of the rows, and only a run "
"managing a trajectory lifecycle ever backfills. Pass fmax, a "
"convergence_hook, or max_trajectory_steps, or drop recycle."
)
if not managed:
return self
if isinstance(self.dynamics, DomainParallel):
raise ValueError(
"A DomainParallel propagator cannot carry a trajectory lifecycle: "
"its step dispatches no ON_GRADUATE stage, so no converged frame "
f"would ever be captured; got dynamics={self.dynamics!r} with "
f"fmax={self.fmax!r}, convergence_hook={self.convergence_hook!r}, "
f"and max_trajectory_steps={self.max_trajectory_steps!r}. Pass "
"the dynamics it wraps, or drop the lifecycle settings."
)
if self.convergence_criterion is not None:
_check_sole_migrator(self.dynamics, self.convergence_criterion)
return self
@model_validator(mode="after")
def _validate_structure_fields(self) -> OnPolicyConfig:
"""Check one row against the propagator's declared keys, its compute(), and the criterion.
The forward pass and the criterion dispatch run only with
``probe=True``, and only once per instance. The after-validators run
again when the config is passed into a strategy, and that second pass
skips both.
"""
probe = self.initial_structures.probe()
self.dynamics.check_initial_batch(probe)
if self._probed or not self.probe:
return self
probed = _probe_propagator(probe, self.dynamics)
criterion = self.convergence_criterion
if probed is not None and criterion is not None:
_probe_criterion(probed, self.dynamics, criterion)
self._probed = True
return self