Source code for nvalchemi.training.distillation.labeling

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Offline labeling of a dataset with teacher signals."""

from __future__ import annotations

import dataclasses
import warnings
from pathlib import Path
from time import perf_counter
from typing import TYPE_CHECKING, Literal, TypeAlias

import torch
import zarr

from nvalchemi.data.datapipes.backends.zarr import (
    AtomicDataZarrReader,
    AtomicDataZarrWriter,
    FieldSchema,
    _get_cat_dim,
)
from nvalchemi.training.distillation._attach import (
    _attach_teacher_labels,
    _prune_empty_edges,
)
from nvalchemi.training.distillation.scoring import (
    _DENSE_NEIGHBOR_KEYS,
    _NEIGHBOR_KEYS,
    _STORABLE_DTYPES,
    _reject_foreign_fields,
    scorer_fields,
)

if TYPE_CHECKING:
    from collections.abc import Iterator, Sequence

    from nvalchemi.data import Batch
    from nvalchemi.data.datapipes.backends.zarr import StoreLike
    from nvalchemi.data.datapipes.dataset import BatchDatasetProtocol
    from nvalchemi.training.distillation.scoring import TeacherScorer

__all__ = ["label_dataset"]


_StoreSchema: TypeAlias = dict[str, FieldSchema]
"""Level, dtype, and row shape of every field a labeled chunk persists, by field."""

_STORE_LEVELS = {"atoms": "atom", "edges": "edge", "system": "system"}
"""Store level names of the built-in batch levels; a custom level keeps its name."""

_PREFETCH_METHODS = ("prefetch_fused_batches", "get_fused_batches", "cancel_prefetch")
"""Dataset methods the pipelined chunk loop reads ahead through."""

_AUTO_PROBE_CHUNKS = 2
"""Chunks ``prefetch="auto"`` reads sequentially before it tries reading ahead."""

_PIPELINE_LOAD_FRACTION = 0.5
"""Load-to-processing time ratio below which ``prefetch="auto"`` stays sequential."""


@dataclasses.dataclass(frozen=True)
class _StoreState:
    """Sample counts and field schema of an existing labeled store."""

    active: int
    total: int
    schema: _StoreSchema


def _row_shape(field: str, shape: Sequence[int]) -> tuple[int, ...]:
    """Return *shape* without the axis a store concatenates *field* along."""
    cat_dim = _get_cat_dim(field) % len(shape)
    return tuple(size for axis, size in enumerate(shape) if axis != cat_dim)


def _existing_store_state(store: StoreLike) -> _StoreState | None:
    """Return the state of *store*, or ``None`` when it cannot be read."""
    try:
        reader = AtomicDataZarrReader(store)
    except (FileNotFoundError, KeyError, ValueError):
        return None
    try:
        reader.check_integrity()
        return _StoreState(
            active=len(reader), total=reader.num_samples, schema=reader.schema()
        )
    finally:
        reader.close()


def _store_holds_data(store: StoreLike) -> bool:
    """Report whether *store* already holds anything a fresh write would destroy.

    A filesystem path is measured by its existence, a mapping store by holding
    a key, and any other store by whether zarr finds a node at its root, since
    an abstract store publishes no synchronous emptiness probe. The question is
    asked of every store type because :meth:`AtomicDataZarrWriter.write` opens
    a new store in mode ``"w"``, which clears whatever it finds there.
    """
    if isinstance(store, (str, Path)):
        return Path(store).exists()
    if isinstance(store, dict):
        return bool(store)
    try:
        zarr.open(store, mode="r")
    except (FileNotFoundError, KeyError, ValueError):
        return False
    return True


def _batch_schema(batch: Batch) -> _StoreSchema:
    """Return the level, dtype, and row shape a writer would persist for each field.

    Mirrors the writer's layout: a system-level tensor has its unit axes after
    the sample axis squeezed away before it is stored. Levels come from the
    batch's storage rather than ``batch.keys``, which carries only the built-in
    levels, so a custom-level field is held to the schema like any other.
    """
    schema: _StoreSchema = {}
    for level, names in batch.level_keys.items():
        for name in names:
            value = batch[name]
            shape = tuple(value.shape)
            if level == "system":
                while len(shape) > 2 and shape[1] == 1:
                    shape = shape[:1] + shape[2:]
            store_level = _STORE_LEVELS.get(level, level)
            schema[name] = FieldSchema(
                store_level, value.dtype, _row_shape(name, shape)
            )
    return schema


def _check_chunk_schema(
    reference: _StoreSchema, outgoing: _StoreSchema, indices: Sequence[int]
) -> None:
    """Raise when a chunk would write a different schema than the store holds.

    The writer's append extends only the arrays a store already holds, so
    drifting fields would misalign arrays, drifting dtypes would cast labels,
    and drifting row shapes would truncate them, all without an error.
    """
    chunk = f"the chunk covering samples {indices[0]!r}-{indices[-1]!r}"
    extra = sorted(set(outgoing) - set(reference))
    missing = sorted(set(reference) - set(outgoing))
    if extra or missing:
        raise ValueError(
            "Every labeled chunk must write the fields the store holds; "
            f"{chunk} writes extra {extra!r} and is missing {missing!r}."
        )
    drifted = ", ".join(
        f"{name} is stored as {reference[name]!r} but arrives as {outgoing[name]!r}"
        for name in sorted(reference)
        if reference[name] != outgoing[name]
    )
    if drifted:
        raise ValueError(
            "Every labeled chunk must write the levels, dtypes, and row shapes the "
            f"store holds; in {chunk}, {drifted}."
        )


def _check_storable_dtypes(outgoing: _StoreSchema) -> None:
    """Raise when a chunk carries a floating-point dtype no store can hold.

    Only the chunk defining a fresh store's schema is checked; every later
    chunk is already held to that schema.
    """
    unstorable = ", ".join(
        f"{name} arrives as {spec.dtype!r}"
        for name, spec in sorted(outgoing.items())
        if spec.dtype.is_floating_point and spec.dtype not in _STORABLE_DTYPES
    )
    if unstorable:
        raise ValueError(
            "Every floating-point field must arrive in a dtype an ALCHEMI Zarr "
            f"store can hold; {unstorable}, and the storable dtypes are "
            f"{list(_STORABLE_DTYPES)!r}."
        )


def _strip_unstorable(
    batch: Batch, keep: frozenset[str], ephemeral: frozenset[str]
) -> None:
    """Drop *ephemeral* and any field that appeared during labeling, keeping *keep*.

    An edge group left with no fields is dropped too, so the store's edge
    pointers never record edges no array backs.
    """
    for key in ephemeral | (frozenset(_batch_schema(batch)) - keep):
        if key in batch:
            del batch[key]
    _prune_empty_edges(batch)


def _chunk_batches(
    dataset: BatchDatasetProtocol,
    chunks: Sequence[list[int]],
    prefetch: bool | Literal["auto"],
) -> Iterator[Batch]:
    """Yield the batch of every chunk, reading one chunk ahead while pipelining is on.

    Under ``prefetch="auto"`` the first :data:`_AUTO_PROBE_CHUNKS` chunks are
    read sequentially, with each load and the caller's processing of the
    yielded batch timed separately. The last of them is the reference: reading
    ahead starts only when its load took at least
    :data:`_PIPELINE_LOAD_FRACTION` of its processing, since a shorter load
    has too little to hide. The first chunk read entirely ahead is then timed
    against the reference per atom, and the run falls back to sequential reads
    when it was not faster, which is what a fast local store looks like once
    the read-ahead thread contends with the scoring. The first chunk is never
    the reference because it carries CUDA warm-up and the store's creation,
    and a chunk without atoms is never a probe because it carries no scoring
    to compare the load against; the next chunk with atoms takes its place. A
    read left pending when the caller stops early is cancelled.
    """
    pipelined = prefetch is True
    pending = False
    probing = prefetch == "auto"
    reference = None
    try:
        for position, indices in enumerate(chunks):
            started = perf_counter()
            read_ahead = pending
            if pending:
                (batch,) = dataset.get_fused_batches()
                pending = False
            else:
                batch = dataset.load_batches([indices])[0]
            if probing and batch.device.type == "cuda":
                torch.cuda.synchronize(batch.device)
            loaded = perf_counter()
            if pipelined and position + 1 < len(chunks):
                dataset.prefetch_fused_batches([chunks[position + 1]])
                pending = True
            yield batch
            # A chunk without atoms carries no scoring, so it cannot be a probe.
            if not probing or position == 0 or batch.num_nodes == 0:
                continue
            elapsed = perf_counter() - started
            if reference is None:
                load = loaded - started
                reference = elapsed / batch.num_nodes
                pipelined = load >= _PIPELINE_LOAD_FRACTION * (elapsed - load)
                probing = pipelined
            elif read_ahead:
                pipelined = elapsed / batch.num_nodes < reference
                probing = False
    finally:
        if pending:
            dataset.cancel_prefetch()


[docs] def label_dataset( dataset: BatchDatasetProtocol, scorer: TeacherScorer, store: StoreLike, *, batch_size: int = 32, device: torch.device | str | None = None, resume: bool = True, keep_neighbors: bool = False, prefetch: bool | Literal["auto"] = "auto", ) -> int: """Label *dataset* with teacher signals and persist the result to *store*. Walks *dataset* in contiguous chunks of *batch_size* samples, scores each chunk with *scorer*, attaches every returned signal as a batch field, and writes the augmented chunk to a Zarr store holding the original fields plus the teacher fields, readable through the ordinary :class:`~nvalchemi.data.datapipes.backends.zarr.AtomicDataZarrReader` / :class:`~nvalchemi.data.datapipes.dataset.Dataset` path. Parameters ---------- dataset : BatchDatasetProtocol Source dataset; only ``__len__`` and ``load_batches`` are used. scorer : TeacherScorer Scorer producing the teacher signals for each chunk. store : StoreLike Destination Zarr store: a path, a zarr store instance, or a dict. batch_size : int, optional Number of samples scored per forward pass. Default ``32``. device : torch.device | str | None, optional Device to move each chunk to before scoring. Default ``None`` (score on whatever device the dataset emits). resume : bool, optional If ``True`` (default), an existing store is treated as a partial run: the first ``len(store)`` samples are skipped and labeling continues from there. A store already holding every sample is a no-op; one holding more samples than *dataset* has is refused. If ``False``, an existing store is an error. keep_neighbors : bool, optional If ``False`` (default), a source neighbor list is dropped rather than stored, because the cutoff it was built at lives on the batch and not in the store. ``True`` carries a sparse (``COO``) source list over; the dense tensors are dropped either way. Default ``False``. prefetch : bool | Literal["auto"], optional Whether to read each chunk while the previous one is scored and written. ``False`` reads, scores, and writes one chunk at a time. ``True`` reads one chunk ahead through the dataset's fused-prefetch surface (``prefetch_fused_batches`` / ``get_fused_batches``), falling back to the sequential loop with a :class:`UserWarning` when the dataset offers none. ``"auto"`` (default) reads the first two chunks sequentially and times the second one's load against the scoring and writing of its chunk; when the load took at least half of that processing it reads ahead, then times the first chunk read entirely ahead against the sequential one per atom and falls back to sequential reads if reading ahead was not faster. A dataset without the surface stays sequential silently. Default ``"auto"``. Returns ------- int Number of samples labeled by this call; ``0`` when a resumed store already covers the whole dataset. Raises ------ ValueError If *batch_size* is not positive, *prefetch* is not ``True``, ``False``, or ``"auto"``, *scorer* declares or returns a batch field outside the ``teacher_*`` namespace, *store* exists but cannot be read as an ALCHEMI Zarr store, *resume* is ``False`` and *store* exists, *store* holds soft-deleted samples or more samples than *dataset* has, *store* holds arrays that disagree about how many samples it contains, a teacher label does not hold one row per atom or per graph, a chunk carries a floating-point field in a dtype a store cannot hold, or a chunk would write a different field set, level, dtype, or row shape than the store holds. TypeError If *scorer* declares ``label_fields`` as a single string. Examples -------- >>> from nvalchemi.training.distillation import label_dataset >>> scorer = InProcessTeacherScorer(teacher, ["energy", "forces"]) # doctest: +SKIP >>> label_dataset(dataset, scorer, "labeled.zarr", batch_size=64) # doctest: +SKIP 1024 Notes ----- The first chunk defines the store schema, and every later chunk — on fresh and resumed runs alike — must write the same fields, levels, dtypes, and row shapes, since the writer would otherwise misalign, cast, or truncate labels silently. Each label is held to the chunk's atom or graph count before it is attached, because the split into per-graph rows would otherwise drop whatever a scorer returned beyond it. Resuming assumes stored sample *i* is dataset sample *i*: soft-deleted samples, a store longer than the dataset, and a store whose arrays disagree with its committed sample count (what an interrupted append leaves) are refused, while drift within the dataset's length is undetectable. Labels are attached with ``overwrite=True``, so a scorer is held to the ``teacher_*`` namespace both by its declared ``label_fields`` and by every chunk it returns, to protect the reference fields it would otherwise replace. Labels stored in float16 or float64 read back at the reading dataset's ``positions`` dtype, because a dataset coerces every floating-point field it loads (:meth:`~nvalchemi.data.AtomicData.check_fp_dtype_consistency`); the stored dtype governs the store's size, not what training sees. Build the student's neighbor list from the stored positions with a :class:`~nvalchemi.hooks.NeighborListHook` at ``BEFORE_FORWARD``. Reading ahead overlaps the next chunk's load with the current chunk's scoring and write, so it saves up to one load per chunk when the store is slow to read (a network or object store, or shared storage) or when per-sample validation dominates the load. The dataset's prefetch thread decodes and validates the chunk while the main thread launches the teacher's kernels, and a dataset that targets a CUDA device also moves every sample there from that thread; on a fast local store the contention can cost more than the load it hides, and labeling runs a little slower than the sequential loop. ``"auto"`` therefore measures both forms on the first chunks rather than assuming; the per-chunk writes, resume bookkeeping, and store contents are the same in every mode. A dataset that emits host-resident chunks, with *device* passed here for the move, keeps the transfer on the main thread and reads ahead faster than one that transfers from the prefetch thread. """ if batch_size <= 0: raise ValueError(f"batch_size must be positive; got {batch_size!r}.") if prefetch not in (True, False, "auto"): raise ValueError(f"prefetch must be True, False, or 'auto'; got {prefetch!r}.") if prefetch is not False and not all( callable(getattr(dataset, name, None)) for name in _PREFETCH_METHODS ): if prefetch is True: warnings.warn( f"{type(dataset).__name__} offers no fused-prefetch surface, so " "labeling reads each chunk sequentially.", UserWarning, stacklevel=2, ) prefetch = False declared = scorer_fields(scorer) if declared is not None: _reject_foreign_fields(declared, "A scorer's label_fields") state = _existing_store_state(store) if state is None and _store_holds_data(store): described = ( str(store) if isinstance(store, (str, Path)) else f"a {type(store).__name__}" ) raise ValueError( "Store already holds data but is not a readable ALCHEMI Zarr store, and " f"writing a fresh one would clear it; got {described}. Label into an " "empty store." ) if state is not None and not resume: raise ValueError( f"Store already exists with {state.active!r} samples and resume is False; " "pass resume=True to continue labeling or write to a fresh store." ) if state is not None and state.active != state.total: raise ValueError( f"Store holds {state.total - state.active!r} soft-deleted samples, so a " "resumed run cannot line up with the dataset; defragment the store or " "label into a fresh one." ) total = len(dataset) start = state.active if state is not None else 0 schema = state.schema if state is not None else None if start > total: raise ValueError( f"Store holds {start!r} samples but the dataset has {total!r}, so it was " "labeled from a different, longer dataset; resume against that dataset " "or label into a fresh store." ) if start == total: return 0 writer = AtomicDataZarrWriter(store) ephemeral = _DENSE_NEIGHBOR_KEYS if keep_neighbors else _NEIGHBOR_KEYS chunks = [ list(range(begin, min(begin + batch_size, total))) for begin in range(start, total, batch_size) ] labeled = 0 for indices, batch in zip(chunks, _chunk_batches(dataset, chunks, prefetch)): if device is not None: batch = batch.to(device) loaded_fields = frozenset(_batch_schema(batch)) labels = scorer.label(batch) _attach_teacher_labels(batch, labels) _strip_unstorable(batch, loaded_fields | frozenset(labels), ephemeral) outgoing = _batch_schema(batch) if schema is None: _check_storable_dtypes(outgoing) writer.write(batch) schema = outgoing else: _check_chunk_schema(schema, outgoing, indices) writer.append(batch) labeled += batch.num_graphs return labeled