Source code for nvalchemi.training.distillation.evaluation.accuracy

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Accuracy, teacher-consistency, and non-conservative diagnostics for students.

:func:`evaluate_accuracy` runs a student over a held-out set. It reports energy,
force, and stress errors against either the reference dataset's own labels or a
teacher's labels. :func:`non_conservative_residual` measures the part of a
teacher's force field that no conservative student can fit. That residual is
the floor against which the accuracy evaluation's force error is read.
"""

from __future__ import annotations

import dataclasses
import functools
import math
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Literal, TypeAlias

import torch
from pydantic import Field

from nvalchemi.data import Batch, resolve_device
from nvalchemi.training import (
    ValidationConfig,
    ValidationLoop,
    default_training_fn,
    ensure_reiterable_validation_data,
    unwrap_model,
)
from nvalchemi.training.distillation.evaluation._export import MeasurementRecord
from nvalchemi.training.distillation.hooks import _score_and_attach
from nvalchemi.training.distillation.scoring import (
    TeacherScorer,
    _as_scorer,
    signal_fields,
)
from nvalchemi.training.distillation.strategy import (
    _student_label_dtype,
    _to_device,
)
from nvalchemi.training.distributed import (
    all_reduce,
    all_reduce_flags,
    is_distributed_initialized,
)
from nvalchemi.training.losses.composition import (
    BaseLossFunction,
    ComposedLossFunction,
)
from nvalchemi.training.losses.reductions import per_graph_sum
from nvalchemi.training.losses.terms import (
    EnergyMSELoss,
    ForceMSELoss,
    StressMSELoss,
)

if TYPE_CHECKING:
    from nvalchemi.models.base import BaseModelMixin

__all__ = [
    "BUILTIN_ACCURACY_QUANTITIES",
    "AccuracyMetrics",
    "AccuracyQuantity",
    "AccuracyQuantitySpec",
    "NonConservativeResidual",
    "evaluate_accuracy",
    "non_conservative_residual",
]

AccuracyQuantity: TypeAlias = Literal["energy", "forces", "stress", "atomic_energies"]
"""Built-in quantity an accuracy evaluation compares between a student and a target."""


[docs] @dataclasses.dataclass(frozen=True) class AccuracyQuantitySpec: """Where one accuracy quantity's prediction and target are read from. The built-in quantities are :data:`BUILTIN_ACCURACY_QUANTITIES`. A custom quantity is passed to :func:`evaluate_accuracy` as an instance, and its errors are reported in ``AccuracyMetrics.errors`` under its name. It can be a head that the training function publishes under its own key, or a built-in quantity read from another field. Parameters ---------- name : str Name the quantity is requested and reported under. prediction_key : str Key of the validation function's output holding the prediction. reference_key : str Batch field holding the dataset's own label. signal : str | None, optional Teacher signal the quantity is compared against when ``targets="teacher"``. The signal is resolved to its batch field through :func:`~nvalchemi.training.distillation.signal_fields`. Default ``None`` (no teacher target; name a teacher field in ``target_keys`` to compare against a teacher). supervised_loss : Callable[..., BaseLossFunction] | None, optional Factory, called as ``supervised_loss(target_key=...)``, that builds the loss term driving the validation pass. Default ``None`` (the quantity is a diagnostic that never enters the pass's loss). Examples -------- >>> from nvalchemi.training.distillation.evaluation import ( ... AccuracyQuantitySpec, ... ) >>> charges = AccuracyQuantitySpec( ... "charges", prediction_key="predicted_charges", reference_key="charges" ... ) >>> charges.supervised_loss is None True """ name: str prediction_key: str reference_key: str signal: str | None = None supervised_loss: Callable[..., BaseLossFunction] | None = None
BUILTIN_ACCURACY_QUANTITIES: Mapping[str, AccuracyQuantitySpec] = { "energy": AccuracyQuantitySpec( "energy", "predicted_energy", "energy", signal="energy", supervised_loss=functools.partial(EnergyMSELoss, per_atom=True), ), "forces": AccuracyQuantitySpec( "forces", "predicted_forces", "forces", signal="forces", supervised_loss=ForceMSELoss, ), "stress": AccuracyQuantitySpec( "stress", "predicted_stress", "stress", signal="stress", supervised_loss=StressMSELoss, ), "atomic_energies": AccuracyQuantitySpec( "atomic_energies", "predicted_atomic_energies", "atomic_energies", signal="atomic_energies", ), } """Specs of the built-in quantities, keyed by the name a caller requests them under.""" _DEFAULT_QUANTITIES: tuple[AccuracyQuantity, ...] = ("energy", "forces") """Quantities evaluated when a caller names none.""" _EPS = 1e-12 """Denominator guard for direction normalization and force-scale ratios.""" _RESIDUAL_SUFFIXES = ("abs", "sq", "count") """Sums accumulated per quantity, in the order :func:`_mae_rmse` reads them.""" _FORCE_ALIGNMENT_KEYS = ( "force_cosine_sum", "force_cosine_count", "force_dot", "force_predicted_sq", "force_target_sq", "force_nonfinite_atoms", ) """Extra sums the force quantity contributes on top of its residuals."""
[docs] class AccuracyMetrics(MeasurementRecord): """Errors of one student against one set of targets over a held-out set. Every metric is an exact global reduction over the evaluated set: the sum of residuals divided by the total count, not a mean of per-batch means. Metrics are in the units the batch carries: eV for energies, eV/A for forces, and the dataset's stress units. A quantity the pass could not measure is ``None``. That happens when the quantity was not requested, or when no batch carried its prediction or target. A quantity that was measured but came out non-finite is ``nan``. An acceptance bar fails a ``nan`` value instead of reporting it as missing. Attributes ---------- name : str Label carried into reports. num_graphs, num_atoms : int Graphs and atoms evaluated. energy_mae, energy_rmse : float | None Total-energy error per graph. energy_per_atom_mae, energy_per_atom_rmse : float | None Total-energy error divided by each graph's atom count. forces_mae, forces_rmse : float | None Force error per Cartesian component over every atom. stress_mae, stress_rmse : float | None Stress error per component over all nine components. force_cosine_mean : float | None Mean over atoms of the cosine between the predicted and target force, weighting every atom equally. An atom whose force is at or below the student's own error has an essentially random angle, so this number describes the held-out set's low-force tail as much as the student. Atoms whose force vanishes on either side are not counted. ``None`` when no atom carries a force on both sides. force_cosine_aggregate : float | None Cosine between the two force fields, each taken as a single vector over the whole set, so atoms are weighted by force magnitude. This is the alignment an acceptance bar reads. A single non-finite atom makes it ``nan``, whereas ``force_cosine_mean`` still reports the atoms that stayed finite. atomic_energies_mae, atomic_energies_rmse : float | None Per-atom energy error, measured when both the prediction and the target include a per-atom energy decomposition. force_nonfinite_atoms : int Atoms dropped from ``force_cosine_mean`` for a non-finite force. errors : Mapping[str, Mapping[str, float]] ``{"mae": ..., "rmse": ...}`` per custom :class:`AccuracyQuantitySpec` that was measured; the built-in quantities report through their own fields. """ name: str num_graphs: int num_atoms: int energy_mae: float | None = None energy_rmse: float | None = None energy_per_atom_mae: float | None = None energy_per_atom_rmse: float | None = None forces_mae: float | None = None forces_rmse: float | None = None stress_mae: float | None = None stress_rmse: float | None = None force_cosine_mean: float | None = None force_cosine_aggregate: float | None = None atomic_energies_mae: float | None = None atomic_energies_rmse: float | None = None force_nonfinite_atoms: int = 0 errors: dict[str, dict[str, float]] = Field(default_factory=dict)
[docs] def to_dict(self) -> dict[str, Any]: """Return the populated fields as a plain dictionary. A quantity that was not measured is left out rather than exported as ``None``, and :meth:`from_dict` restores it as ``None``. An empty ``errors`` map is left out the same way. """ exported = self.model_dump(exclude_none=True) if not exported["errors"]: del exported["errors"] return exported
[docs] class NonConservativeResidual(MeasurementRecord): r"""Non-conservative component of a teacher's force field. The *non-conservative residual* is the part of a teacher's force field that no conservative student can fit. A force field decomposes as :math:`F = -\nabla E + F_{\perp}`. A student that predicts forces as the gradient of an energy represents only the first term, whose work around any closed path vanishes. A closed-path integral of the teacher's field therefore measures :math:`F_{\perp}` alone. Every force here is a root-mean-square per-atom magnitude in eV/A. Compare ``force_floor`` with the ``forces_rmse`` of :class:`AccuracyMetrics` accordingly: ``forces_rmse`` is a per-component figure, smaller by ``sqrt(3)`` for an isotropic error. Attributes ---------- num_probes : int Closed loops integrated in total, ``num_loops`` per graph. amplitude : float Loop side length in A, as a root-mean-square per-atom displacement. segments : int Midpoint-rule samples per side. loop_work_mean_abs, loop_work_max_abs : float Mean and maximum absolute work around one loop, in eV. force_floor, force_floor_max : float Mean and maximum lower bound the loop work places on a conservative student's root-mean-square per-atom force error, in eV/A. force_rms : float Root-mean-square teacher force at the loop centers over every atom. relative_floor, relative_floor_max : float Mean and maximum of each probe's floor divided by the root-mean-square teacher force of its own graph. A batch that mixes force scales therefore reports a figure between its graphs' ratios. A graph at equilibrium is divided by a small clamp rather than by zero. """ num_probes: int amplitude: float segments: int loop_work_mean_abs: float loop_work_max_abs: float force_floor: float force_floor_max: float force_rms: float relative_floor: float relative_floor_max: float
class _PlacedBatches: """Re-iterable view that places each batch on *device* and labels it if scored. Every batch is placed before the loop sees it, whatever device it starts on. The placement is a clone: a copy into device memory is asynchronous, a copy into host memory blocks so the moved batch's pointers can be read at once, and a batch already on the device is cloned without a transfer. Because it is a clone, a scorer's ``teacher_*`` fields stay off the caller's batches. The scorer labels after the placement, at the precision its own ``autocast`` setting selects, and a label outside ``teacher_*`` is refused. """ def __init__( self, source: Iterable[Batch], device: torch.device, scorer: TeacherScorer | None = None, ) -> None: self.source = source self.device = device self.scorer = scorer def __iter__(self) -> Iterator[Batch]: """Yield each source batch on the run device, labeled if a scorer was given.""" for batch in self.source: placed = _to_device(batch, self.device) if self.scorer is not None: _score_and_attach(self.scorer, placed) yield placed class _MetricAccumulator: """Per-batch callback accumulating exact residual sums over a validation pass. Implements :class:`~nvalchemi.training.BatchValidationCallback` and runs inside a :class:`ValidationLoop` pass. Sums are float64 device tensors, reduced once at the end. Every sum the requested quantities can produce is seeded at zero up front. The packed all-reduce tensor therefore has the same shape and key order on every rank, even when a shard carried no target for some quantity. """ def __init__( self, device: torch.device, quantities: Sequence[AccuracyQuantitySpec], target_keys: Mapping[str, str], ) -> None: self.device = device self.specs = tuple(quantities) self.quantities = tuple(spec.name for spec in self.specs) self.target_keys = dict(target_keys) self._sums: dict[str, torch.Tensor] = { key: torch.zeros((), device=device, dtype=torch.float64) for key in _metric_keys(self.quantities) } def __call__( self, *, batch: Batch, predictions: Mapping[str, torch.Tensor], loss: Any, # noqa: ARG002 batch_count: int, # noqa: ARG002 step_count: int, # noqa: ARG002 epoch: int, # noqa: ARG002 ) -> None: """Accumulate one validation batch's residual sums.""" self._add("num_graphs", batch.num_graphs) self._add("num_atoms", batch.num_nodes) for spec in self.specs: quantity = spec.name prediction = predictions.get(spec.prediction_key) target = getattr(batch, self.target_keys[quantity], None) if prediction is None or target is None: continue if quantity == "atomic_energies": self._accumulate( quantity, prediction.reshape(-1), target.reshape(-1), target.numel() ) continue self._accumulate(quantity, prediction, target, target.numel()) if quantity == "energy": counts = batch.num_nodes_per_graph.reshape( (-1,) + (1,) * (target.ndim - 1) ) self._accumulate( "energy_per_atom", prediction / counts, target / counts, target.shape[0], ) elif quantity == "forces": self._accumulate_cosine(prediction, target) def _add(self, key: str, value: torch.Tensor | float) -> None: """Add one contribution to the running float64 sum *key*.""" tensor = value.detach() if isinstance(value, torch.Tensor) else value scalar = torch.as_tensor(tensor, device=self.device, dtype=torch.float64) previous = self._sums.get(key) self._sums[key] = scalar if previous is None else previous + scalar def _accumulate( self, name: str, prediction: torch.Tensor, target: torch.Tensor, count: int ) -> None: """Add the absolute and squared residual sums of one quantity. Shapes must match exactly. Broadcasting a ``(B,)`` target against a ``(B, 1)`` prediction would silently measure every pairing. """ if prediction.shape != target.shape: raise ValueError( f"Prediction and target of {name!r} must have the same shape; got " f"{tuple(prediction.shape)!r} and {tuple(target.shape)!r}." ) residual = prediction.detach().to(torch.float64) - target.detach().to( torch.float64 ) self._add(f"{name}_abs", residual.abs().sum()) self._add(f"{name}_sq", residual.pow(2).sum()) self._add(f"{name}_count", float(count)) def _accumulate_cosine( self, prediction: torch.Tensor, target: torch.Tensor ) -> None: """Add the per-atom and aggregate force-alignment sums. The ``> 0.0`` test only guards against zero divided by zero, so a small force still counts at full weight in the per-atom mean. A non-finite atom is also dropped from the per-atom mean and is counted in ``force_nonfinite_atoms``. It stays in the aggregate sums, so the whole-set alignment reports as unmeasurable rather than as an average over the atoms that are left. """ predicted = prediction.detach().to(torch.float64) reference = target.detach().to(torch.float64) dot = (predicted * reference).sum(dim=-1) predicted_norm = predicted.norm(dim=-1) reference_norm = reference.norm(dim=-1) finite = torch.isfinite(predicted).all(dim=-1) & torch.isfinite(reference).all( dim=-1 ) aligned = finite & (predicted_norm > 0.0) & (reference_norm > 0.0) norms = predicted_norm * reference_norm self._add("force_cosine_sum", (dot[aligned] / norms[aligned]).sum()) self._add("force_cosine_count", float(aligned.sum())) self._add("force_nonfinite_atoms", float((~finite).sum())) self._add("force_dot", dot.sum()) self._add("force_predicted_sq", predicted.pow(2).sum()) self._add("force_target_sq", reference.pow(2).sum()) def metrics(self, *, name: str, distributed_manager: Any | None) -> AccuracyMetrics: """Return the reduced metrics, all-reducing sums under distributed runs. Whether this rank measured anything is decided locally and exchanged with :func:`~nvalchemi.training.distributed.all_reduce_flags` before the sums are reduced. A rank whose shard carried no target still enters the all-reduce, contributing zeros, because the metrics are global ratios of the summed residuals and counts. Only when every rank came up empty is there nothing to divide, and then every rank raises the same refusal rather than leaving a peer waiting in the all-reduce. Raises ------ ValueError If no rank measured any quantity, meaning every batch on every rank was missing either the prediction or the target of every requested quantity. """ keys = tuple(sorted(self._sums)) packed = torch.stack([self._sums[key] for key in keys]) measured = any( float(self._sums[key]) > 0.0 for key in keys if key.endswith("_count") ) empty = all_reduce_flags(not measured, distributed_manager) if bool(empty.all()): raise ValueError( "No accuracy metric could be measured on any rank; every batch was " "missing the prediction or the target of every requested quantity " f"{list(self.quantities)!r} (targets {self.target_keys!r})." ) if is_distributed_initialized(distributed_manager): all_reduce(packed, distributed_manager) totals = {key: float(packed[index]) for index, key in enumerate(keys)} energy_mae, energy_rmse = _mae_rmse(totals, "energy") per_atom_mae, per_atom_rmse = _mae_rmse(totals, "energy_per_atom") forces_mae, forces_rmse = _mae_rmse(totals, "forces") stress_mae, stress_rmse = _mae_rmse(totals, "stress") atomic_mae, atomic_rmse = _mae_rmse(totals, "atomic_energies") errors = {} for quantity in self.quantities: if quantity in BUILTIN_ACCURACY_QUANTITIES: continue mae, rmse = _mae_rmse(totals, quantity) if mae is not None: errors[quantity] = {"mae": mae, "rmse": rmse} return AccuracyMetrics( name=name, num_graphs=int(totals.get("num_graphs", 0.0)), num_atoms=int(totals.get("num_atoms", 0.0)), energy_mae=energy_mae, energy_rmse=energy_rmse, energy_per_atom_mae=per_atom_mae, energy_per_atom_rmse=per_atom_rmse, forces_mae=forces_mae, forces_rmse=forces_rmse, stress_mae=stress_mae, stress_rmse=stress_rmse, force_cosine_mean=_ratio( totals.get("force_cosine_sum"), totals.get("force_cosine_count") ), force_cosine_aggregate=_aggregate_cosine(totals), atomic_energies_mae=atomic_mae, atomic_energies_rmse=atomic_rmse, force_nonfinite_atoms=int(totals.get("force_nonfinite_atoms", 0.0)), errors=errors, ) def _metric_keys(quantities: Sequence[str]) -> tuple[str, ...]: """Return every sum the requested *quantities* can contribute to.""" keys = ["num_graphs", "num_atoms"] for quantity in quantities: prefixes = ["energy_per_atom", quantity] if quantity == "energy" else [quantity] keys.extend( f"{prefix}_{suffix}" for prefix in prefixes for suffix in _RESIDUAL_SUFFIXES ) if quantity == "forces": keys.extend(_FORCE_ALIGNMENT_KEYS) return tuple(keys) def _mae_rmse( totals: Mapping[str, float], prefix: str ) -> tuple[float | None, float | None]: """Return the MAE and RMSE of one quantity, or two ``None`` when unmeasured.""" count = totals.get(f"{prefix}_count", 0.0) if count <= 0.0: return None, None return totals[f"{prefix}_abs"] / count, math.sqrt(totals[f"{prefix}_sq"] / count) def _ratio(numerator: float | None, denominator: float | None) -> float | None: """Return ``numerator / denominator``, or ``None`` when either is missing. A denominator of zero also yields ``None``. """ if numerator is None or not denominator: return None return numerator / denominator def _aggregate_cosine(totals: Mapping[str, float]) -> float | None: """Return the cosine similarity of the two force fields taken as one vector. A non-finite sum means the alignment was measured but is unusable, so it reports ``nan`` for a bar to fail. A norm of exactly zero means every force in the set vanishes, so it reports ``None``, like any quantity that was not measured. """ dot = totals.get("force_dot") if dot is None: return None squares = totals["force_predicted_sq"] * totals["force_target_sq"] if not math.isfinite(dot) or not math.isfinite(squares): return math.nan norm = math.sqrt(squares) return dot / norm if norm > 0.0 else None def _teacher_target_keys(specs: Sequence[AccuracyQuantitySpec]) -> dict[str, str]: """Return the teacher field each quantity with a signal is compared against. Each signal is resolved to its field through the scoring module, so if a signal changes the field it writes, the evaluation follows it. A spec without a signal has no teacher field and is left out. Raises ------ RuntimeError If the signal behind a quantity publishes more than one field. """ resolved: dict[str, str] = {} for spec in specs: if spec.signal is None: continue fields = signal_fields([spec.signal]) if len(fields) != 1: raise RuntimeError( f"Quantity {spec.name!r} is compared against a single teacher " f"field, but its signal {spec.signal!r} publishes {list(fields)!r}." ) resolved[spec.name] = fields[0] return resolved def _resolve_quantities( quantities: Sequence[str | AccuracyQuantitySpec] | None, ) -> tuple[AccuracyQuantitySpec, ...]: """Return the specs *quantities* request, built-ins looked up by name. Raises ------ ValueError If a name is not a built-in quantity, or two specs share a name. """ if quantities is None: return tuple(BUILTIN_ACCURACY_QUANTITIES[name] for name in _DEFAULT_QUANTITIES) unknown = sorted( quantity for quantity in quantities if isinstance(quantity, str) and quantity not in BUILTIN_ACCURACY_QUANTITIES ) if unknown: raise ValueError( "Accuracy quantities must be names from " f"{sorted(BUILTIN_ACCURACY_QUANTITIES)!r} or AccuracyQuantitySpec " f"instances; got unsupported {unknown!r}." ) specs = tuple( BUILTIN_ACCURACY_QUANTITIES[quantity] if isinstance(quantity, str) else quantity for quantity in quantities ) names = [spec.name for spec in specs] if len(set(names)) != len(names): raise ValueError(f"Accuracy quantity names must be unique; got {names!r}.") return specs def _resolve_device(model: Any, device: torch.device | str | None) -> torch.device: """Return the requested device, else the device the model's parameters sit on. An explicit request goes through :func:`~nvalchemi.data.resolve_device`, so an index-less ``"cuda"`` names the current CUDA device and compares equal to the device the placed batches report. """ if device is not None: return resolve_device(device) parameters = getattr(model, "parameters", None) if callable(parameters): for parameter in parameters(): return parameter.device return torch.device("cpu") def _requires_autograd(model: Any) -> bool: """Return whether a model's own forward pass needs autograd enabled around it. It reads :attr:`~nvalchemi.models.base.BaseModelMixin.requires_autograd` through :func:`~nvalchemi.training.unwrap_model`, so a parallelism wrapper around the student does not hide the declaration. A model without the attribute is taken to publish direct outputs only. """ return bool(getattr(unwrap_model(model), "requires_autograd", False)) def _metric_loss( specs: Sequence[AccuracyQuantitySpec], target_keys: Mapping[str, str] ) -> ComposedLossFunction: """Build the composed loss whose gradient requirement drives the pass.""" terms = [ spec.supervised_loss(target_key=target_keys[spec.name]) for spec in specs if spec.supervised_loss is not None ] if not terms: raise ValueError( "At least one quantity carrying a supervised loss ('energy', 'forces', " "or 'stress' among the built-ins) must be evaluated so the validation " f"pass has a loss to run; got {[spec.name for spec in specs]!r}." ) return ComposedLossFunction(terms, dtype_policy="prediction_to_target")
[docs] def evaluate_accuracy( model: BaseModelMixin, data: Iterable[Batch], *, targets: Literal["reference", "teacher"] = "reference", quantities: Sequence[str | AccuracyQuantitySpec] | None = None, scorer: TeacherScorer | BaseModelMixin | None = None, target_keys: Mapping[str, str] | None = None, loss_fn: ComposedLossFunction | None = None, validation_fn: Callable[..., Any] = default_training_fn, grad_mode: Literal["auto", "enabled", "disabled"] = "auto", label_dtype: torch.dtype | None = None, device: torch.device | str | None = None, distributed_manager: Any | None = None, name: str = "accuracy", ) -> AccuracyMetrics: """Measure a student's error over a held-out set. The pass runs through :class:`~nvalchemi.training.ValidationLoop`, so eval mode and device placement behave as in training validation. The autograd policy is settled before the loop runs, because a student that differentiates inside its own forward needs gradients whatever is scored. The loop is built without an autocast context, so the student predicts in its own dtype. A bare *scorer* model labels with autocast disabled, the in-process scorer's default; a supplied scorer decides its own precision. The metrics are exact global residual sums accumulated in float64. The loop's graph-balanced loss value is discarded. ``targets="reference"`` compares against the dataset's own labels. ``targets="teacher"`` compares against the ``teacher_*`` fields, which :func:`~nvalchemi.training.distillation.label_dataset` writes offline or a *scorer* writes on the fly. Against either family, the force-alignment numbers fill in whenever forces are compared, and the per-atom energy diagnostic whenever ``"atomic_energies"`` is requested and both sides carry it. A *scorer* passed with reference targets is refused, because its teacher pass would be paid for and thrown away. Parameters ---------- model : BaseModelMixin Student to evaluate. The call leaves it in the training mode it arrived in, and scores exactly the weights passed. For a student trained under an ``EMAHook``, pass ``strategy.inference_model``. Nothing downstream can tell which weights were scored, so record the choice on :class:`~nvalchemi.training.distillation.evaluation.StudentEvaluation`. data : Iterable[Batch] Re-iterable held-out set. One-shot iterators are rejected. targets : {"reference", "teacher"}, optional Family of batch fields to compare against. Default ``"reference"``. quantities : Sequence[str | AccuracyQuantitySpec] | None, optional Quantities to evaluate. Each entry is a built-in name from :data:`BUILTIN_ACCURACY_QUANTITIES`, or an :class:`AccuracyQuantitySpec` instance for a custom quantity, which is reported under ``errors``. The built-in ``"atomic_energies"`` is a diagnostic that never enters the pass's loss. Default ``None`` (energy and forces). scorer : TeacherScorer | BaseModelMixin | None, optional Teacher that labels each batch before it is evaluated. A bare model is wrapped in an :class:`~nvalchemi.training.distillation.InProcessTeacherScorer` for the requested quantities, and its labels are cast to *label_dtype*. A supplied scorer's labels are not cast. Default ``None``. target_keys : Mapping[str, str] | None, optional Per-quantity overrides of the batch field to compare against. They take precedence over the fields *targets* selects. Default ``None``. loss_fn : ComposedLossFunction | None, optional Loss driving the pass. Default ``None`` (mean-squared terms over the requested supervised quantities). validation_fn : Callable, optional Forward callable invoked as ``validation_fn(model, batch)``. Default :func:`~nvalchemi.training.default_training_fn`. grad_mode : {"auto", "enabled", "disabled"}, optional Autograd policy. ``"auto"`` enables gradients whenever the student's forward needs them or the loss does. ``"disabled"`` is refused for a student whose forward differentiates. Default ``"auto"``. label_dtype : torch.dtype | None, optional Dtype that a bare *scorer* model's labels are cast to. Default ``None`` (the dtype the distillation strategy infers for the student's own labels: the dtype of its first floating-point parameter, floored at float32). device : torch.device | str | None, optional Device the pass runs on. Default ``None`` (the model's own device). distributed_manager : Any | None, optional Manager used to all-reduce the metric sums. Default ``None``. name : str, optional Label stored on the result. Default ``"accuracy"``. Returns ------- AccuracyMetrics Errors and consistency diagnostics over the whole set. Raises ------ ValueError If *quantities* names an unknown quantity, repeats a name, or includes no supervised quantity. If a quantity without a teacher signal is compared against the teacher and has no ``target_keys`` entry. If a *scorer* is given but no requested quantity is compared against a teacher field, or the scorer does not publish the fields the evaluation reads, or it returns a label outside ``teacher_*``. If gradients are disabled for a student that differentiates inside its forward. If a prediction and its target disagree on shape, or if no metric could be measured on any rank. If any rank received no batches: the refusal is raised on every rank, naming the empty ones, before the pass starts. Examples -------- >>> from nvalchemi.training.distillation.evaluation import evaluate_accuracy >>> metrics = evaluate_accuracy(student, holdout) # doctest: +SKIP >>> metrics.forces_mae # doctest: +SKIP 0.031 Against the teacher, labeling on the fly: >>> metrics = evaluate_accuracy( # doctest: +SKIP ... student, ... holdout, ... targets="teacher", ... scorer=teacher, ... quantities=("energy", "forces", "atomic_energies"), ... ) Notes ----- The student is called through *validation_fn* exactly as a training loop would call it. A student that reads a neighbor list therefore needs batches that carry one. A *scorer* builds the teacher's own list for each batch and rolls it back afterwards. ``grad_mode="auto"`` reads the student's ``autograd_outputs``, the same declaration that :meth:`~nvalchemi.models.base.BaseModelMixin.adapt_input` reads, and looks through any ``DistributedDataParallel`` or similar wrapper to find it. An autograd-force student is therefore scored with gradients even when only energies are compared. Narrowing its ``active_outputs`` puts it back on the ``torch.no_grad()`` path. Under a distributed run, every rank must call this function with the same *quantities* and a non-empty shard. The sums are packed in one key order before the all-reduce, so packs of different shapes would deadlock. Whether each rank has a batch is exchanged before the pass, because the validation loop raises on an empty shard before its own summary reduce and would strand the other ranks there; the refusal names the empty ranks on every rank instead. Shard sizes, and the targets each shard carries, may differ: a shard with no target for a quantity contributes zeros to its sums. """ specs = _resolve_quantities(quantities) requested = tuple(spec.name for spec in specs) teacher_keys = _teacher_target_keys(specs) base = ( teacher_keys if targets == "teacher" else {spec.name: spec.reference_key for spec in specs} ) resolved_keys = dict(base) | dict(target_keys or {}) unresolved = sorted(set(requested) - set(resolved_keys)) if unresolved: raise ValueError( f"Quantities {unresolved!r} declare no teacher signal, so " f"targets={targets!r} has no field to compare them against; name " "the teacher field in target_keys or give the spec a signal." ) compared = {resolved_keys[quantity] for quantity in requested} if scorer is not None and not (compared & set(teacher_keys.values())): raise ValueError( f"A scorer was given, but targets={targets!r} compares against " f"{sorted(compared)!r}, none of which is a teacher field. The teacher " "pass would be paid and thrown away, and the reported errors would be " "against the dataset's own labels rather than the teacher's. Pass " "targets='teacher', drop the scorer, or name the teacher fields to " "compare against in target_keys." ) differentiated = _requires_autograd(model) if differentiated and grad_mode == "disabled": raise ValueError( f"Student {type(model).__name__!r} computes an active output by " "differentiating inside its own forward, so it needs autograd " "whatever is being scored; got grad_mode='disabled'. Pass " "grad_mode='auto', or narrow the student's model_config.active_outputs " "so it stops differentiating." ) resolved_device = _resolve_device(model, device) reiterable = ensure_reiterable_validation_data(data) empty = all_reduce_flags(next(iter(reiterable), None) is None, distributed_manager) if bool(empty.any()): raise ValueError( "evaluate_accuracy received no batches on rank(s) " f"{empty.nonzero().flatten().tolist()!r}. Under a distributed run the " "sums are all-reduced across ranks, so every rank needs at least one " "batch of the held-out set; shard it so none comes up empty." ) signals = [spec.signal for spec in specs if spec.signal is not None] evaluation_data: Iterable[Batch] = _PlacedBatches( reiterable, resolved_device, scorer=( _as_scorer( scorer, signals, _student_label_dtype(model) if label_dtype is None else label_dtype, ) if scorer is not None else None ), ) accumulator = _MetricAccumulator(resolved_device, specs, resolved_keys) config = ValidationConfig( validation_data=evaluation_data, loss_fn=loss_fn or _metric_loss(specs, resolved_keys), grad_mode=grad_mode, batch_callback=accumulator, name=name, ) loop = ValidationLoop( validation_data=evaluation_data, config=config, device=resolved_device, model=model, validation_fn=validation_fn, grad_enabled=True if differentiated else None, distributed_manager=distributed_manager, ) with loop: loop.execute() return accumulator.metrics(name=name, distributed_manager=distributed_manager)
@contextmanager def _displaced(batch: Batch, positions: torch.Tensor) -> Iterator[None]: """Swap *positions* onto *batch* for the block, restoring the originals after.""" original = batch.positions batch.positions = positions try: yield finally: batch.positions = original def _probe_directions( batch: Batch, generator: torch.Generator | None ) -> tuple[torch.Tensor, torch.Tensor]: """Return two per-graph orthogonal displacement directions for *batch*. Each direction has unit root-mean-square per-atom norm within each graph. Scaling a direction by ``amplitude`` therefore moves the atoms by ``amplitude`` in root-mean-square, whatever the graph's size. Normalizing by the graph's Frobenius norm instead would shrink the step as ``1 / sqrt(N)``. """ positions = batch.positions device = positions.device if generator is None else generator.device shape = tuple(positions.shape) first = torch.randn(shape, generator=generator, device=device).to(positions) second = torch.randn(shape, generator=generator, device=device).to(positions) index = batch.batch_idx sizes = batch.num_nodes_per_graph.to(positions) overlap = per_graph_sum( (first * second).sum(dim=-1), batch.batch_idx, num_graphs=batch.num_graphs ) norm = per_graph_sum( first.pow(2).sum(dim=-1), batch.batch_idx, num_graphs=batch.num_graphs ) second = second - (overlap / norm.clamp_min(_EPS))[index].unsqueeze(-1) * first first = first / (norm / sizes).sqrt().clamp_min(_EPS)[index].unsqueeze(-1) second_norm = ( per_graph_sum( second.pow(2).sum(dim=-1), batch.batch_idx, num_graphs=batch.num_graphs ) / sizes ) second = second / second_norm.sqrt().clamp_min(_EPS)[index].unsqueeze(-1) return first, second
[docs] def non_conservative_residual( teacher: TeacherScorer | BaseModelMixin, data: Iterable[Batch] | Batch, *, num_loops: int = 4, amplitude: float = 0.05, segments: int = 4, generator: torch.Generator | None = None, ) -> NonConservativeResidual: r"""Estimate the part of a teacher's force field no conservative student can fit. For each graph, two orthogonal directions :math:`u` and :math:`v`, each with unit root-mean-square per-atom norm, span a square loop of side *amplitude* :math:`\varepsilon` through configuration space. The teacher's work :math:`W = \oint F \cdot \mathrm{d}R` around the loop is integrated with the midpoint rule at *segments* samples per side. A conservative field integrates to zero, so :math:`W` measures the non-conservative component alone. The loop's path length is :math:`4 \varepsilon \sqrt{N}`, so by Cauchy-Schwarz :math:`|W| / (4 \varepsilon N)` is a lower bound on the largest root-mean-square per-atom force error a conservative student makes on that loop. This bound is reported as ``force_floor``. The floor is a bound at the probed displacement scale, not a dataset-wide error bar. It shrinks linearly with *amplitude*, so choose an amplitude on the order of a thermal vibration. One randomly oriented loop sees a :math:`1 / \sqrt{3N}` fraction of the field's curl, so floors are comparable only between probes of similar system size. A conservative teacher reports the midpoint rule's quadrature error and, below that, the round-off of the batch's own dtype. A floor below the float32 round-off plateau needs a float64 batch and teacher. See :ref:`distillation-evaluation` for the magnitudes. Parameters ---------- teacher : TeacherScorer | BaseModelMixin Teacher whose field is probed. A bare model is wrapped in an :class:`~nvalchemi.training.distillation.InProcessTeacherScorer`, which builds and rolls back the teacher's neighbor list at each probe point. data : Iterable[Batch] | Batch Held-out structures to probe. Positions are displaced in place and restored before returning. num_loops : int, optional Loops integrated per graph. Default ``4``. amplitude : float, optional Loop side length in A, as a per-atom displacement. Default ``0.05``. segments : int, optional Midpoint-rule samples per side; one loop costs ``4 * segments`` teacher force evaluations. Default ``4``. generator : torch.Generator | None, optional Generator drawing the loop directions. Default ``None`` (the global RNG). Returns ------- NonConservativeResidual Loop work, the force floor it implies, and its size relative to the teacher's own force scale. Raises ------ ValueError If *amplitude*, *num_loops*, or *segments* is not positive, if the scorer does not publish the teacher force field, or if *data* holds no graphs. Examples -------- >>> from nvalchemi.training.distillation.evaluation import ( ... non_conservative_residual, ... ) >>> residual = non_conservative_residual(teacher, holdout) # doctest: +SKIP >>> residual.relative_floor # doctest: +SKIP 0.02 """ if amplitude <= 0.0 or num_loops <= 0 or segments <= 0: raise ValueError( "amplitude, num_loops, and segments must all be positive; got " f"amplitude={amplitude!r}, num_loops={num_loops!r}, " f"segments={segments!r}." ) scorer = _as_scorer(teacher, ["forces"]) works: list[torch.Tensor] = [] sizes: list[torch.Tensor] = [] graph_scales: list[torch.Tensor] = [] force_squares: list[torch.Tensor] = [] for batch in [data] if isinstance(data, Batch) else data: squares = ( scorer.label(batch)["teacher_forces"][0] .pow(2) .sum(dim=-1) .flatten() .to(torch.float64) ) force_squares.append(squares) base = batch.positions counts = batch.num_nodes_per_graph.to(torch.float64) scale = ( per_graph_sum(squares, batch.batch_idx, num_graphs=batch.num_graphs) / counts ).sqrt() for _ in range(num_loops): first, second = _probe_directions(batch, generator) works.append( _loop_work(scorer, batch, base, first, second, amplitude, segments) ) sizes.append(counts) graph_scales.append(scale) if not works: raise ValueError("data must hold at least one graph to probe.") work = torch.cat(works).abs().to(torch.float64) floor = work / (4.0 * amplitude * torch.cat(sizes)) relative = floor / torch.cat(graph_scales).clamp_min(_EPS) magnitudes = torch.cat(force_squares) return NonConservativeResidual( num_probes=int(work.numel()), amplitude=amplitude, segments=segments, loop_work_mean_abs=float(work.mean()), loop_work_max_abs=float(work.max()), force_floor=float(floor.mean()), force_floor_max=float(floor.max()), force_rms=float(magnitudes.mean().sqrt()), relative_floor=float(relative.mean()), relative_floor_max=float(relative.max()), )
def _loop_work( scorer: Any, batch: Batch, base: torch.Tensor, first: torch.Tensor, second: torch.Tensor, amplitude: float, segments: int, ) -> torch.Tensor: """Integrate the teacher's work around one closed rectangular loop. The samples nearly cancel, so they are accumulated in float64. The probe points are laid out around each graph's own centroid. The precision with which a displaced position is represented, and so the floor a conservative teacher reports, then does not depend on where in space a graph sits or how far it lies from the other graphs in the batch. """ counts = batch.num_nodes_per_graph.to(base).unsqueeze(-1) centered = ( base - (per_graph_sum(base, batch.batch_idx, num_graphs=batch.num_graphs) / counts)[ batch.batch_idx ] ) corners = ( torch.zeros_like(first), amplitude * first, amplitude * (first + second), amplitude * second, ) work = base.new_zeros(batch.num_graphs, dtype=torch.float64) for index in range(4): start = corners[index] step = (corners[(index + 1) % 4] - start) / segments for sample in range(segments): with _displaced(batch, centered + (start + step * (sample + 0.5))): forces = scorer.label(batch)["teacher_forces"][0] contribution = (forces.to(torch.float64) * step.to(torch.float64)).sum( dim=-1 ) work = work + per_graph_sum( contribution, batch.batch_idx, num_graphs=batch.num_graphs ) return work