# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Acceptance verdicts across a family of students.
A caller collects one :class:`StudentEvaluation` per candidate student and sets
the acceptance bars on an :class:`AcceptanceThresholds`.
:func:`build_acceptance_report` then returns an :class:`AcceptanceReport`, which
renders as Rich tables and exports as a plain dictionary. :func:`measured_bars`
reports which bars a partial set of measurements can decide. Every measurement
rebuilds from its own export, so a sweep can evaluate each student in a
separate job and assemble the report in a final job.
"""
from __future__ import annotations
import dataclasses
import math
from collections.abc import Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias, get_args
from pydantic import BaseModel, ConfigDict, Field, model_validator
from rich import box
from rich.console import Group
from rich.table import Table
from nvalchemi.training.distillation.evaluation._export import MeasurementRecord
from nvalchemi.training.distillation.evaluation.accuracy import (
AccuracyMetrics,
AccuracyQuantity,
)
from nvalchemi.training.distillation.evaluation.stability import (
ExtensivityMetrics,
RDFComparison,
StabilityMetrics,
)
__all__ = [
"AcceptanceBar",
"AcceptanceCheck",
"AcceptanceReport",
"AcceptanceThresholds",
"BAR_FAMILIES",
"DEFAULT_BARS",
"MetricFamily",
"StudentEvaluation",
"StudentVerdict",
"build_acceptance_report",
"measured_bars",
]
_MISSING = "-"
"""Cell rendered where a student has no value for a column."""
_WEIGHT_SOURCES = ("ema", "raw")
"""Weight sets a student evaluation can record having been measured on."""
_EXTRA_FAMILY_PREFIX = "extra:"
"""Prefix naming a family a bar reads out of ``StudentEvaluation.extra``."""
_COMPARISONS = ("<=", ">=")
"""Directions an acceptance bar can pass in."""
_FROM_SCRATCH_BAR = "max_from_scratch_ratio"
"""The one bar formed across two families, and so the one with no ``check``."""
_DIVERGENCE_BAR = "no_divergence"
"""The stability bar read as a flag rather than compared against a number."""
MetricFamily: TypeAlias = Literal[
"accuracy",
"stability",
"extensivity",
"rdf",
"baseline_accuracy",
]
"""Measurement slot of a :class:`StudentEvaluation` an acceptance bar reads."""
_STUDENT_SECTIONS: dict[MetricFamily, type[MeasurementRecord]] = {
"accuracy": AccuracyMetrics,
"stability": StabilityMetrics,
"extensivity": ExtensivityMetrics,
"rdf": RDFComparison,
"baseline_accuracy": AccuracyMetrics,
}
"""Measurement class behind each nested slot of a student evaluation."""
[docs]
class StudentEvaluation(MeasurementRecord):
"""Everything measured about one candidate student.
Only *name* and *accuracy* are required. Each remaining slot holds a
measurement that a caller may or may not have run. A bar aimed at an empty
slot fails the student rather than passing it silently. *weights* is not a
slot: it notes where the numbers came from, and no bar reads it.
Attributes
----------
name : str
Label the student is reported under.
accuracy : AccuracyMetrics
Held-out errors, from
:func:`~nvalchemi.training.distillation.evaluation.evaluate_accuracy`.
stability : StabilityMetrics | None
Trajectory conservation metrics, from
:meth:`~nvalchemi.training.distillation.evaluation.StabilityMonitor.metrics`.
extensivity : ExtensivityMetrics | None
Energy-scaling error, from
:func:`~nvalchemi.training.distillation.evaluation.extensivity_error`.
rdf : RDFComparison | None
Structural match against a reference trajectory, from
:func:`~nvalchemi.training.distillation.evaluation.compare_radial_distributions`.
baseline_accuracy : AccuracyMetrics | None
The same accuracy evaluation run on an equal-size student trained from
scratch. The from-scratch bar compares the student against it. A
baseline whose graph and atom counts differ from those of *accuracy*
fails the bar rather than being used for a ratio.
num_parameters : int | None
Parameter count, reported beside the student's errors.
weights : Literal["ema", "raw"] | None
Which of the student's weights the numbers were measured on: its
EMA-averaged weights (``"ema"``) or its live ones (``"raw"``). Only the
caller knows whether it passed
:func:`~nvalchemi.training.distillation.evaluation.evaluate_accuracy` a
``strategy.inference_model``, so record it here. ``None`` records
nothing, which is not the same as ``"raw"``.
extra : Mapping[str, Mapping[str, float]]
Measurements outside the typed slots, as one flat map of numbers per
family. A custom :class:`AcceptanceBar` reads a family through
``"extra:<family>"``. Default empty.
Raises
------
TypeError
If a measurement slot holds anything but its own metrics class.
ValueError
If *weights* names neither of the two weight sets.
"""
name: str
accuracy: AccuracyMetrics
stability: StabilityMetrics | None = None
extensivity: ExtensivityMetrics | None = None
rdf: RDFComparison | None = None
baseline_accuracy: AccuracyMetrics | None = None
num_parameters: int | None = None
weights: Literal["ema", "raw"] | None = None
extra: dict[str, dict[str, float]] = Field(default_factory=dict)
@model_validator(mode="before")
@classmethod
def _check_slots(cls, data: Any) -> Any:
"""Reject a mistyped measurement slot or an unrecognized weights marker.
The slots are read attribute by attribute much later, when the report
is built. Without this check, an object of the wrong kind would surface
as an ``AttributeError`` inside :func:`build_acceptance_report` rather
than at the line that filled the slot. A mapping is let through for
field validation to rebuild.
"""
if not isinstance(data, Mapping):
return data
for slot, metric in _STUDENT_SECTIONS.items():
value = data.get(slot)
if value is not None and not isinstance(value, (metric, Mapping)):
raise TypeError(
f"StudentEvaluation.{slot} must be a {metric.__name__} or "
f"None; got {value!r}. An accessor left uncalled, such as "
"StabilityMonitor.metrics rather than the metrics it "
"returns, is the usual cause."
)
weights = data.get("weights")
if weights is not None and weights not in _WEIGHT_SOURCES:
raise ValueError(
f"StudentEvaluation.weights must be one of {list(_WEIGHT_SOURCES)!r} "
f"or None; got {weights!r}."
)
return data
[docs]
def to_dict(self) -> dict[str, Any]:
"""Return the populated measurements and markers as plain dictionaries."""
measured = {
"name": self.name,
"accuracy": self.accuracy.to_dict(),
"stability": self.stability,
"extensivity": self.extensivity,
"rdf": self.rdf,
"baseline_accuracy": self.baseline_accuracy,
"num_parameters": self.num_parameters,
"weights": self.weights,
"extra": self.extra or None,
}
return {
key: value.to_dict() if hasattr(value, "to_dict") else value
for key, value in measured.items()
if value is not None
}
[docs]
@classmethod
def from_dict(cls, data: Mapping[str, Any]) -> StudentEvaluation:
"""Rebuild an evaluation, and its measurements, from a :meth:`to_dict` export.
An entry taken straight out of :meth:`AcceptanceReport.to_dict` is also
accepted. Its ``verdict`` is dropped, because verdicts are formed from
the thresholds of the report being built, not carried between jobs.
Raises
------
ValueError
If the export, or one of its nested measurements, carries a key the
record does not declare or omits a required one.
"""
rebuilt = {key: value for key, value in data.items() if key != "verdict"}
for key, metric in _STUDENT_SECTIONS.items():
if rebuilt.get(key) is not None:
rebuilt[key] = metric.from_dict(rebuilt[key])
return super().from_dict(rebuilt)
[docs]
class AcceptanceThresholds(BaseModel):
"""Acceptance bars a student has to clear to be accepted.
Every bar defaults to ``None``, which means "do not test this", except
``no_divergence``, a flag that defaults to ``False`` with the same
meaning. A bar that is set but has no matching measurement fails the
student; it is never skipped silently. A bar outside the built-in table is
set through ``extra``, keyed by the name of its :class:`AcceptanceBar`.
:func:`build_acceptance_report` refuses an ``extra`` key that its table of
bars does not carry.
Examples
--------
>>> from nvalchemi.training.distillation.evaluation import AcceptanceThresholds
>>> thresholds = AcceptanceThresholds(
... max_energy_per_atom_mae=0.005,
... max_forces_mae=0.05,
... max_energy_drift_per_atom_per_ns=0.01,
... max_from_scratch_ratio=1.0,
... )
>>> thresholds.max_from_scratch_ratio
1.0
"""
max_energy_per_atom_mae: Annotated[
float | None,
Field(default=None, gt=0, description="Largest accepted energy MAE per atom."),
] = None
max_forces_mae: Annotated[
float | None,
Field(
default=None,
gt=0,
description="Largest accepted force MAE per Cartesian component.",
),
] = None
max_stress_mae: Annotated[
float | None,
Field(default=None, gt=0, description="Largest accepted stress MAE."),
] = None
min_force_cosine: Annotated[
float | None,
Field(
default=None,
ge=-1.0,
le=1.0,
description=(
"Smallest accepted magnitude-weighted cosine similarity between "
"the student's and the target force fields. It is read off "
"force_cosine_aggregate rather than the per-atom mean, which the "
"holdout's near-zero forces dominate."
),
),
] = None
max_energy_drift_per_atom_per_ns: Annotated[
float | None,
Field(
default=None,
gt=0,
description="Largest accepted fitted energy drift rate, in eV/atom/ns.",
),
] = None
max_energy_drift_per_atom_per_step: Annotated[
float | None,
Field(
default=None,
gt=0,
description="Largest accepted energy drift per atom per step.",
),
] = None
max_momentum_drift: Annotated[
float | None,
Field(
default=None,
gt=0,
description=(
"Largest accepted deviation of a graph's total momentum. Only "
"meaningful under a momentum-conserving integrator: a stochastic "
"thermostat exchanges momentum with its bath by design."
),
),
] = None
no_divergence: Annotated[
bool,
Field(
default=False,
description=(
"Fail a student whose stability series stopped on a divergence, "
"that is, whose StabilityMetrics carry a first_divergence_step. "
"The drift bars alone cannot catch one: the series ends at the "
"step the monitor's predicate fired, so they describe the segment "
"before it, which can look perfectly conserved."
),
),
] = False
max_extensivity_error_per_atom: Annotated[
float | None,
Field(
default=None,
gt=0,
description="Largest accepted supercell energy-scaling error per atom.",
),
] = None
max_rdf_jensen_shannon: Annotated[
float | None,
Field(
default=None,
gt=0,
le=1.0,
description=(
"Largest accepted Jensen-Shannon divergence between the student's "
"and the reference trajectory's pair-distance histograms. Blind "
"to which species a pair joins unless the compared distributions "
"were themselves resolved to one pair of species."
),
),
] = None
max_from_scratch_ratio: Annotated[
float | None,
Field(
default=None,
gt=0,
description=(
"Largest accepted ratio of the distilled student's error to the "
"equal-size from-scratch student's error on the same holdout. The "
"ratio is taken on energy_per_atom_mae, forces_mae, and stress_mae, "
"whichever both carry, and the worst one is kept. A limit of 1.0 "
"demands a match and a limit below 1.0 demands a margin."
),
),
] = None
extra: Annotated[
dict[str, float],
Field(
default_factory=dict,
description=(
"Limits of custom acceptance bars, keyed by the AcceptanceBar name "
"they are applied under in the table the report is built with."
),
),
]
model_config = ConfigDict(extra="forbid")
[docs]
@dataclasses.dataclass(frozen=True)
class AcceptanceBar:
"""Where one acceptance bar reads the number it gates, and which way it passes.
An *acceptance bar* is a limit that one number from a student's
measurements must clear for the student to be accepted. The built-in bars
are :data:`DEFAULT_BARS`. To register a custom bar, pass a table that
includes it to :func:`build_acceptance_report` and :func:`measured_bars`.
Set its limit under its name in ``AcceptanceThresholds.extra``, and file
its number under its family in ``StudentEvaluation.extra``.
Parameters
----------
name : str
Name the bar's limit is set under: a field of
:class:`AcceptanceThresholds` for a built-in bar, or a key of its
``extra`` for a custom one.
families : tuple[str, ...]
Measurement slots the bar reads. A slot of :class:`StudentEvaluation`
names a typed measurement. ``"extra:<family>"`` names a map of numbers
under ``StudentEvaluation.extra``.
check : str, optional
Row the bar reports under. Unless *attribute* overrides it, this is
also the field or key the bar reads from its first family. Required
for every bar but the built-in from-scratch ratio, which spans two
families and is computed by the report itself. Default ``""``.
attribute : str, optional
Field or key read when it differs from *check*. Default ``""``.
comparison : {"<=", ">="}, optional
Direction the check passes in. Default ``"<="``.
quantities : tuple[str, ...], optional
Accuracy quantities, any one of which is enough to decide the bar.
:func:`measured_bars` uses them to narrow its answer. Default ``()``.
missing : str, optional
Detail reported when the family was supplied but the number it reads
was not. Default ``""``.
Raises
------
ValueError
If *name* or *families* is empty, if *check* is empty for any bar but
the from-scratch ratio, if a family is neither a measurement slot nor
an ``"extra:"`` family, or if *comparison* is not ``"<="`` or
``">="``.
Examples
--------
>>> from nvalchemi.training.distillation.evaluation import (
... DEFAULT_BARS,
... AcceptanceBar,
... )
>>> dipole_bar = AcceptanceBar(
... "max_dipole_mae", ("extra:dipole",), "dipole_mae"
... )
>>> bars = (*DEFAULT_BARS, dipole_bar)
"""
name: str
families: tuple[str, ...]
check: str = ""
attribute: str = ""
comparison: Literal["<=", ">="] = "<="
quantities: tuple[str, ...] = ()
missing: str = ""
def __post_init__(self) -> None:
"""Reject a bar that names nothing, reads nowhere, or passes in no direction."""
if not self.name or not self.families:
raise ValueError(
"An acceptance bar needs a name and at least one family; got "
f"name={self.name!r}, families={self.families!r}."
)
if not self.check and self.name != _FROM_SCRATCH_BAR:
raise ValueError(
f"Acceptance bar {self.name!r} names no field or key to check. Only "
f"the built-in {_FROM_SCRATCH_BAR!r}, which the report forms across "
"two families, leaves check empty."
)
unknown = [family for family in self.families if not _is_family(family)]
if unknown:
raise ValueError(
f"Acceptance bar {self.name!r} reads unknown families {unknown!r}; "
f"expected slots from {sorted(_STUDENT_SECTIONS)!r} or "
f"{_EXTRA_FAMILY_PREFIX!r} followed by a family name."
)
if self.comparison not in _COMPARISONS:
raise ValueError(
f"Acceptance bar {self.name!r} must compare with one of "
f"{list(_COMPARISONS)!r}; got {self.comparison!r}."
)
def _is_family(family: str) -> bool:
"""Return whether *family* names a typed slot or a non-empty extra family."""
return family in _STUDENT_SECTIONS or (
family.startswith(_EXTRA_FAMILY_PREFIX)
and len(family) > len(_EXTRA_FAMILY_PREFIX)
)
DEFAULT_BARS: tuple[AcceptanceBar, ...] = (
AcceptanceBar(
"max_energy_per_atom_mae",
("accuracy",),
"energy_per_atom_mae",
quantities=("energy",),
missing="the accuracy pass did not compare energy",
),
AcceptanceBar(
"max_forces_mae",
("accuracy",),
"forces_mae",
quantities=("forces",),
missing="the accuracy pass did not compare forces",
),
AcceptanceBar(
"max_stress_mae",
("accuracy",),
"stress_mae",
quantities=("stress",),
missing="the accuracy pass did not compare stress",
),
AcceptanceBar(
"min_force_cosine",
("accuracy",),
"force_cosine_aggregate",
comparison=">=",
quantities=("forces",),
missing="the accuracy pass did not compare forces",
),
AcceptanceBar(
"max_energy_drift_per_atom_per_ns",
("stability",),
"energy_drift_per_atom_per_ns",
missing="the trajectory was recorded without a timestep, so no rate was fitted",
),
AcceptanceBar(
"max_energy_drift_per_atom_per_step",
("stability",),
"energy_drift_per_atom_per_step",
),
AcceptanceBar("max_momentum_drift", ("stability",), "max_momentum_drift"),
AcceptanceBar(
_DIVERGENCE_BAR, ("stability",), "divergence", "first_divergence_step"
),
AcceptanceBar(
"max_extensivity_error_per_atom",
("extensivity",),
"extensivity_error_per_atom",
"max_error_per_atom",
),
AcceptanceBar(
"max_rdf_jensen_shannon", ("rdf",), "rdf_jensen_shannon", "jensen_shannon"
),
AcceptanceBar(
_FROM_SCRATCH_BAR,
("accuracy", "baseline_accuracy"),
quantities=("energy", "forces", "stress"),
),
)
"""Built-in bars of :class:`AcceptanceThresholds`, in the order they are applied."""
BAR_FAMILIES: Mapping[str, frozenset[str]] = {
bar.name: frozenset(bar.families) for bar in DEFAULT_BARS
}
"""Measurement families each built-in bar reads, keyed by threshold field."""
_BUILTIN_BAR_FIELDS = frozenset(BAR_FAMILIES)
"""Threshold fields the built-in bars are set through, as opposed to ``extra``."""
[docs]
def measured_bars(
*families: str,
accuracy_quantities: Sequence[AccuracyQuantity] | None = None,
bars: Sequence[AcceptanceBar] = DEFAULT_BARS,
) -> frozenset[str]:
"""Return the acceptance bars *families* hold enough measurements to decide.
A bar counts as measured only when every family it reads was supplied,
because :func:`build_acceptance_report` fails a student on a bar whose
measurement is missing rather than skipping it. ``max_from_scratch_ratio``
therefore needs both ``"accuracy"`` and ``"baseline_accuracy"``.
*accuracy_quantities* narrows the answer further. An accuracy pass fills
only the quantities it compared, so a held-out set scored on energy alone
leaves ``max_forces_mae`` as unfillable as no pass at all. One bar has a
precondition that no argument here can express, and it is reported on the
strength of its family alone: ``max_energy_drift_per_atom_per_ns`` needs a
:class:`~nvalchemi.training.distillation.evaluation.StabilityMonitor` built
with ``timestep_fs``. A check that fails for that reason says so.
Parameters
----------
*families : str
Slots of a :class:`StudentEvaluation` the caller fills, or
``"extra:<family>"`` for a number map under its ``extra``. Naming none
returns an empty set.
accuracy_quantities : Sequence[AccuracyQuantity] | None, optional
Quantities the accuracy pass compared. Default ``None`` (every
quantity).
bars : Sequence[AcceptanceBar], optional
Table of bars to answer for. Default :data:`DEFAULT_BARS`.
Returns
-------
frozenset[str]
Names of the bars that may be set: fields of
:class:`AcceptanceThresholds`, or keys of its ``extra``.
Raises
------
ValueError
If a name in *families* is not a measurement family the table reads,
or a name in *accuracy_quantities* is not an accuracy quantity.
Examples
--------
>>> from nvalchemi.training.distillation.evaluation import measured_bars
>>> sorted(measured_bars("accuracy"))
['max_energy_per_atom_mae', 'max_forces_mae', 'max_stress_mae', 'min_force_cosine']
>>> sorted(measured_bars("accuracy", accuracy_quantities=["energy"]))
['max_energy_per_atom_mae']
"""
supplied = frozenset(families)
readable = set(_STUDENT_SECTIONS).union(*(bar.families for bar in bars))
unknown = sorted(supplied - readable)
if unknown:
raise ValueError(
f"Unknown measurement families {unknown!r}; expected names from "
f"{sorted(readable)!r}."
)
known = frozenset(get_args(AccuracyQuantity))
if accuracy_quantities is None:
compared = known
else:
compared = frozenset(accuracy_quantities)
unknown = sorted(compared - known)
if unknown:
raise ValueError(
f"Unknown accuracy quantities {unknown!r}; expected names from "
f"{sorted(known)!r}."
)
return frozenset(
bar.name
for bar in bars
if frozenset(bar.families) <= supplied
and (not bar.quantities or compared & frozenset(bar.quantities))
)
[docs]
@dataclasses.dataclass(frozen=True)
class AcceptanceCheck:
"""One threshold applied to one measurement.
Attributes
----------
name : str
Row label of the check: the bar's ``check`` name, or
``"from_scratch_ratio"`` for the baseline bar. It can differ from the
field read, as ``extensivity_error_per_atom`` reads
``max_error_per_atom``.
value : float | None
Measured value, or ``None`` when the measurement is missing. The
``divergence`` row reports ``1.0`` when the stability series stopped
on a divergence and ``0.0`` when it never did, against a limit of
``0.0``.
limit : float | None
Bar the value was compared against.
comparison : {"<=", ">="}
Direction the check passes in.
passed : bool
Whether the student cleared the bar.
detail : str
Why a check failed, when the reason is not the number itself.
"""
name: str
value: float | None
limit: float | None
comparison: Literal["<=", ">="]
passed: bool
detail: str = ""
[docs]
def to_dict(self) -> dict[str, Any]:
"""Return every field as a plain dictionary."""
return dataclasses.asdict(self)
[docs]
@dataclasses.dataclass(frozen=True)
class StudentVerdict:
"""Outcome of every check applied to one student.
Attributes
----------
name : str
Student the verdict belongs to.
accepted : bool
``True`` when every check passed. A student with no checks at all is
accepted, since no bar was asked for.
checks : tuple[AcceptanceCheck, ...]
Checks in the order they were applied.
"""
name: str
accepted: bool
checks: tuple[AcceptanceCheck, ...]
[docs]
def to_dict(self) -> dict[str, Any]:
"""Return the verdict and its checks as plain dictionaries."""
return {
"name": self.name,
"accepted": self.accepted,
"checks": [check.to_dict() for check in self.checks],
}
[docs]
@dataclasses.dataclass(frozen=True)
class AcceptanceReport:
"""Verdicts, the student summary, and the exports a workflow logs.
The report is a final artifact rather than a stream, so it does not
implement the :class:`~nvalchemi.hooks.Reporter` protocol. Print it to a
:class:`rich.console.Console` for the dashboard view. Pass :meth:`scalars`
to a :class:`~nvalchemi.hooks.TensorBoardReporter`, or to any scalar sink,
for a durable record.
Attributes
----------
thresholds : AcceptanceThresholds
Bars the verdicts were formed against.
evaluations : tuple[StudentEvaluation, ...]
Measurements the report was built from, in the order supplied.
verdicts : tuple[StudentVerdict, ...]
One verdict per evaluation, aligned with ``evaluations``.
Examples
--------
>>> from rich.console import Console
>>> from nvalchemi.training.distillation.evaluation import (
... build_acceptance_report,
... )
>>> report = build_acceptance_report(evaluations, thresholds) # doctest: +SKIP
>>> Console().print(report) # doctest: +SKIP
>>> report.accepted # doctest: +SKIP
True
"""
thresholds: AcceptanceThresholds
evaluations: tuple[StudentEvaluation, ...]
verdicts: tuple[StudentVerdict, ...]
@property
def accepted(self) -> bool:
"""Return whether every student cleared every bar it was given."""
return all(verdict.accepted for verdict in self.verdicts)
[docs]
def to_dict(self) -> dict[str, Any]:
"""Return the whole report as nested plain dictionaries."""
return {
"accepted": self.accepted,
"thresholds": self.thresholds.model_dump(exclude_none=True),
"students": [
evaluation.to_dict() | {"verdict": verdict.to_dict()}
for evaluation, verdict in zip(
self.evaluations, self.verdicts, strict=True
)
],
}
[docs]
def scalars(self) -> dict[str, float]:
"""Return a flat ``{student/group/metric: value}`` map of every number.
Returns
-------
dict[str, float]
Numeric metrics only, keyed for a scalar sink such as
:class:`~nvalchemi.hooks.TensorBoardReporter`. Verdicts appear as
``<student>/accepted`` with value ``1.0`` or ``0.0``. A top-level
number such as ``num_parameters`` appears as
``<student>/num_parameters``.
"""
flat: dict[str, float] = {}
for evaluation, verdict in zip(self.evaluations, self.verdicts, strict=True):
flat[f"{evaluation.name}/accepted"] = float(verdict.accepted)
for group, metrics in evaluation.to_dict().items():
if not isinstance(metrics, dict):
if isinstance(metrics, (int, float)) and not isinstance(
metrics, bool
):
flat[f"{evaluation.name}/{group}"] = float(metrics)
continue
for key, value in metrics.items():
if isinstance(value, dict):
for inner, number in value.items():
flat[f"{evaluation.name}/{group}/{key}/{inner}"] = float(
number
)
elif isinstance(value, (int, float)) and not isinstance(
value, bool
):
flat[f"{evaluation.name}/{group}/{key}"] = float(value)
return flat
def __rich__(self) -> Group:
"""Render the verdict and summary tables as one renderable."""
return Group(_verdict_table(self.verdicts), _summary_table(self))
def _format(value: float | None) -> str:
"""Return a compact cell for an optional number."""
return _MISSING if value is None else f"{value:.4g}"
def _finite(value: float | None) -> bool:
"""Return whether *value* is a number a bar can be decided from."""
return value is not None and math.isfinite(value)
def _check(
name: str,
value: float | None,
limit: float | None,
comparison: Literal["<=", ">="],
detail: str = "",
missing: str = "not measured",
) -> AcceptanceCheck | None:
"""Return the check for one bar, or ``None`` when no bar was set.
A missing measurement reports *missing*, a measured value carries *detail*,
and a non-finite value fails on ``"not finite"``: a NaN would read as an
ordinary miss, ``-inf`` would pass every ``max_*`` bar, and ``+inf`` every
``min_*`` bar.
"""
if limit is None:
return None
if not _finite(value):
return AcceptanceCheck(
name=name,
value=value,
limit=limit,
comparison=comparison,
passed=False,
detail=missing if value is None else "not finite",
)
passed = value <= limit if comparison == "<=" else value >= limit
return AcceptanceCheck(
name=name,
value=value,
limit=limit,
comparison=comparison,
passed=passed,
detail=detail,
)
def _baseline_check(
evaluation: StudentEvaluation, limit: float | None
) -> AcceptanceCheck | None:
"""Return the from-scratch check: the student must match or beat its baseline.
The ratio is taken on ``energy_per_atom_mae``, ``forces_mae``, and
``stress_mae``, whichever both carry, and the worst one is kept. A
baseline scored on a different number of graphs or atoms fails this
student's own check, not the report for the whole family. A baseline error
of exactly zero is unbeatable: a matching student ties at ``1.0``, and any
error fails at infinity. A non-finite error on either side yields no ratio
at all.
"""
if limit is None:
return None
baseline = evaluation.baseline_accuracy
if baseline is None:
return AcceptanceCheck(
name="from_scratch_ratio",
value=None,
limit=limit,
comparison="<=",
passed=False,
detail="no from-scratch baseline supplied",
)
student_workload = (evaluation.accuracy.num_graphs, evaluation.accuracy.num_atoms)
baseline_workload = (baseline.num_graphs, baseline.num_atoms)
if student_workload != baseline_workload:
return AcceptanceCheck(
name="from_scratch_ratio",
value=None,
limit=limit,
comparison="<=",
passed=False,
detail=(
f"baseline scored {baseline_workload!r} against the student's "
f"{student_workload!r} as (graphs, atoms)"
),
)
ratios: dict[str, float] = {}
for field in ("energy_per_atom_mae", "forces_mae", "stress_mae"):
error = getattr(evaluation.accuracy, field)
reference = getattr(baseline, field)
if error is None or reference is None:
continue
if not _finite(error) or not _finite(reference):
ratios[field] = math.nan
elif reference == 0.0:
ratios[field] = 1.0 if error == 0.0 else math.inf
else:
ratios[field] = error / reference
if not ratios:
return AcceptanceCheck(
name="from_scratch_ratio",
value=None,
limit=limit,
comparison="<=",
passed=False,
detail="baseline shares no comparable accuracy metric",
)
unusable = sorted(field for field, ratio in ratios.items() if math.isnan(ratio))
if unusable:
return AcceptanceCheck(
name="from_scratch_ratio",
value=math.nan,
limit=limit,
comparison="<=",
passed=False,
detail=f"no finite ratio for {unusable!r}",
)
worst = max(ratios.values())
return AcceptanceCheck(
name="from_scratch_ratio",
value=worst,
limit=limit,
comparison="<=",
passed=worst <= limit,
detail="worst error ratio against the equal-size from-scratch student",
)
def _divergence_check(
evaluation: StudentEvaluation, enabled: bool
) -> AcceptanceCheck | None:
"""Return the divergence check: a series that stopped on a divergence fails.
The flag is reported as a number so the row reads like any maximum: ``1.0``
when :class:`~nvalchemi.dynamics.hooks.StabilityMetrics` carry a
``first_divergence_step`` and ``0.0`` when they do not, against a limit of
``0.0``. Like every bar, it fails a student with no stability record.
"""
if not enabled:
return None
stability = evaluation.stability
if stability is None:
return AcceptanceCheck(
name="divergence",
value=None,
limit=0.0,
comparison="<=",
passed=False,
detail="not measured",
)
step = stability.first_divergence_step
return AcceptanceCheck(
name="divergence",
value=float(step is not None),
limit=0.0,
comparison="<=",
passed=step is None,
detail="" if step is None else f"a graph diverged at step {step}",
)
def _rdf_detail(comparison: RDFComparison | None) -> str:
"""Return which pair distribution an RDF bar was measured over."""
if comparison is None or comparison.pair is None:
return "species-blind total g(r)"
return f"partial g(r) of atomic numbers {list(comparison.pair)!r}"
def _limit(thresholds: AcceptanceThresholds, bar: AcceptanceBar) -> float | bool | None:
"""Return the limit set for *bar*: its own field, else its ``extra`` entry."""
if bar.name in _BUILTIN_BAR_FIELDS:
return getattr(thresholds, bar.name)
return thresholds.extra.get(bar.name)
def _family_metrics(evaluation: StudentEvaluation, family: str) -> Any:
"""Return the measurement *family* names on *evaluation*, or ``None``."""
if family.startswith(_EXTRA_FAMILY_PREFIX):
return evaluation.extra.get(family[len(_EXTRA_FAMILY_PREFIX) :])
return getattr(evaluation, family)
def _read(metrics: Any, attribute: str) -> float | None:
"""Return *attribute* off a typed measurement or out of a number map."""
if isinstance(metrics, Mapping):
return metrics.get(attribute)
return getattr(metrics, attribute)
def _student_checks(
evaluation: StudentEvaluation,
thresholds: AcceptanceThresholds,
bars: Sequence[AcceptanceBar],
) -> tuple[AcceptanceCheck, ...]:
"""Apply every bar of *bars* that *thresholds* sets to one student.
Each bar reads its measurement through its first family. A bar absent from
*bars* is neither applied nor reported. A bar whose family was measured but
whose own number was not reports which quantity or timestep was missing.
Two built-in bars are formed by the report rather than read: the
from-scratch ratio, which :func:`_baseline_check` takes across two
families, and the divergence flag, which :func:`_divergence_check` reads
off the stability record.
"""
candidates = []
for bar in bars:
limit = _limit(thresholds, bar)
if bar.name == _FROM_SCRATCH_BAR:
candidates.append(_baseline_check(evaluation, limit))
continue
if bar.name == _DIVERGENCE_BAR:
candidates.append(_divergence_check(evaluation, bool(limit)))
continue
family = bar.families[0]
metrics = _family_metrics(evaluation, family)
candidates.append(
_check(
bar.check,
None if metrics is None else _read(metrics, bar.attribute or bar.check),
limit,
bar.comparison,
_rdf_detail(evaluation.rdf) if family == "rdf" else "",
"not measured" if metrics is None else bar.missing or "not measured",
)
)
return tuple(check for check in candidates if check is not None)
def _verdict_table(verdicts: Sequence[StudentVerdict]) -> Table:
"""Build the per-check verdict table."""
table = Table(title="Acceptance", box=box.SIMPLE_HEAD, expand=True)
for column in ("Student", "Check", "Value", "Bar", "Result"):
table.add_column(column)
for verdict in verdicts:
if not verdict.checks:
table.add_row(verdict.name, "no bars set", _MISSING, _MISSING, "ACCEPT")
continue
for index, check in enumerate(verdict.checks):
table.add_row(
verdict.name if index == 0 else "",
check.name if not check.detail else f"{check.name} ({check.detail})",
_format(check.value),
f"{check.comparison} {_format(check.limit)}",
"pass" if check.passed else "FAIL",
)
return table
def _summary_table(report: AcceptanceReport) -> Table:
"""Build the one-row-per-student summary across the family.
The verdict column cannot wrap, so a narrow console abbreviates a header
rather than truncating the ``ACCEPT`` or ``REJECT`` the reader is looking
for.
"""
table = Table(title="Students", box=box.SIMPLE_HEAD, expand=True)
for column in ("Student", "Params", "E/atom MAE", "F MAE"):
table.add_column(column)
table.add_column("Verdict", no_wrap=True)
for evaluation, verdict in zip(report.evaluations, report.verdicts, strict=True):
table.add_row(
evaluation.name,
_MISSING
if evaluation.num_parameters is None
else f"{evaluation.num_parameters:,}",
_format(evaluation.accuracy.energy_per_atom_mae),
_format(evaluation.accuracy.forces_mae),
"ACCEPT" if verdict.accepted else "REJECT",
)
return table
[docs]
def build_acceptance_report(
evaluations: Sequence[StudentEvaluation],
thresholds: AcceptanceThresholds | None = None,
*,
bars: Sequence[AcceptanceBar] = DEFAULT_BARS,
) -> AcceptanceReport:
"""Turn a family of student evaluations into verdicts.
Parameters
----------
evaluations : Sequence[StudentEvaluation]
One evaluation per candidate student. Names must be unique, since they
key the report's exports.
thresholds : AcceptanceThresholds | None, optional
Bars to apply. Default ``None`` (no bars: every student is accepted and
the report is a comparison table).
bars : Sequence[AcceptanceBar], optional
Table of bars to apply. Extend :data:`DEFAULT_BARS` to gate a
measurement filed under ``StudentEvaluation.extra``. Default
:data:`DEFAULT_BARS`.
Returns
-------
AcceptanceReport
Verdicts aligned with *evaluations*.
Raises
------
ValueError
If *evaluations* is empty, or two students share a name. If two
entries of *bars* share a name. If ``thresholds.extra`` sets a limit
for a bar that the table does not carry, or for a built-in bar that is
set through its own field. If the students were not all scored on the
same held-out set.
Examples
--------
>>> from nvalchemi.training.distillation.evaluation import (
... AcceptanceThresholds,
... StudentEvaluation,
... build_acceptance_report,
... )
>>> report = build_acceptance_report( # doctest: +SKIP
... [StudentEvaluation(name="student-s", accuracy=metrics)],
... AcceptanceThresholds(max_forces_mae=0.05),
... )
>>> report.accepted # doctest: +SKIP
True
"""
if not evaluations:
raise ValueError("At least one student evaluation is required to report on.")
names = [evaluation.name for evaluation in evaluations]
if len(set(names)) != len(names):
raise ValueError(f"Student names must be unique; got {names!r}.")
resolved = thresholds if thresholds is not None else AcceptanceThresholds()
bar_names = [bar.name for bar in bars]
if len(set(bar_names)) != len(bar_names):
raise ValueError(f"Acceptance bar names must be unique; got {bar_names!r}.")
custom = set(bar_names) - _BUILTIN_BAR_FIELDS
unplaced = sorted(set(resolved.extra) - custom)
if unplaced:
raise ValueError(
f"AcceptanceThresholds.extra sets {unplaced!r}, but the table applies "
f"custom bars {sorted(custom)!r} only; a built-in bar is set through "
"its own field, and a custom one needs its AcceptanceBar in bars."
)
holdouts = {
(evaluation.accuracy.num_graphs, evaluation.accuracy.num_atoms)
for evaluation in evaluations
}
if len(holdouts) > 1:
raise ValueError(
f"Students were scored on different holdouts {sorted(holdouts)!r} as "
"(num_graphs, num_atoms). Errors are comparable only across students "
"scored on one holdout. Re-run evaluate_accuracy for every student "
"over the same held-out data."
)
verdicts = []
for evaluation in evaluations:
checks = _student_checks(evaluation, resolved, bars)
verdicts.append(
StudentVerdict(
name=evaluation.name,
accepted=all(check.passed for check in checks),
checks=checks,
)
)
return AcceptanceReport(
thresholds=resolved,
evaluations=tuple(evaluations),
verdicts=tuple(verdicts),
)