Source code for nvalchemi.data.level_storage

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Unified tensor-based data containers for ML workflows on atomic simulations.

This module provides efficient, torch-only data management for molecular dynamics
and quantum chemistry calculations. Data is organized by **level**: per-atom (node),
per-edge (bond/neighbor), and per-system (graph) tensors.

Data model
----------

Conceptually, a batched atomic dataset is a set of tensors split by level. Each
level has either a **uniform** layout (one row per system, e.g. system-level
fields like ``cell``, ``energy``) or a **segmented** layout (concatenated
variable-length segments, e.g. per-atom ``positions``, per-edge ``neighbor_list``).
The following diagram shows how the classes fit together:

.. graphviz::
   :caption: MultiLevelStorage class hierarchy.

   digraph level_storage {
       rankdir=TB
       fontname="Helvetica"
       node [fontname="Helvetica" fontsize=11 shape=box style="rounded,filled" fillcolor="#dce6f1"]
       edge [fontname="Helvetica" fontsize=10]

       schema [label="LevelSchema\n(maps tensor names → level, dtype,\nsegmentation flag; schema only)" fillcolor="#f9e2ae"]
       multi  [label="MultiLevelStorage\n(one container per level;\nroutes name → container)"]

       schema -> multi [label="used by" style=bold]

       seg_atoms [label="SegmentedLevelStorage\n(atoms — variable len)"]
       seg_edges [label="SegmentedLevelStorage\n(edges — variable len)"]
       uni_sys   [label="UniformLevelStorage\n(system — one per system)"]

       multi -> seg_atoms [label="atoms" style=bold]
       multi -> seg_edges [label="edges" style=bold]
       multi -> uni_sys   [label="system" style=bold]

       base [label="BaseLevelStorage\n(ABC: one level, dict-like)" fillcolor="#eeeeee"]

       seg_atoms -> base [label="inherits" style=dashed]
       seg_edges -> base [label="inherits" style=dashed]
       uni_sys   -> base [label="inherits" style=dashed]
   }

Classes
-------
LevelSchema
    Registry that maps attribute names to levels (groups), dtypes, and
    segmentation flags. In chemistry terms: which level (atom / edge / system)
    each tensor belongs to.
BaseLevelStorage
    Abstract base for a single level: dict-like container of tensors with
    uniform or segmented layout.
UniformLevelStorage
    One level with **uniform** first dimension (e.g. system-level: one row
    per system).
SegmentedLevelStorage
    One level with **segmented** layout: concatenated variable-length segments
    and batch pointer bookkeeping (e.g. atom-level, edge-level).
MultiLevelStorage
    Multi-level container: holds ``BaseLevelStorage`` instances keyed by level
    (e.g. ``"atoms"``, ``"edges"``, ``"system"``) and routes attribute access.

The class names use a **Level** / **Storage** convention: ``LevelSchema`` for the
registry, ``BaseLevelStorage`` for the abstract single-level container, and
``UniformLevelStorage`` / ``SegmentedLevelStorage`` / ``MultiLevelStorage`` for
the concrete implementations.
"""

from __future__ import annotations

import contextlib
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import Collection, Iterator
from typing import Any, Literal, TypeAlias

import numpy as np
import torch
import warp as wp
from tensordict import TensorDict
from torch import Tensor

from nvalchemi.data.buffer_kernels import (
    TORCH_TO_WP,
    compute_put_fit_mask_per_system,
    defrag_per_system,
    defrag_segmented,
    put_masked_per_system,
    put_masked_segmented,
)

wp.config.quiet = True
try:
    wp.init()
except RuntimeError as e:
    raise RuntimeError(
        "Failed to initialize warp, likely due to missing drivers and/or devices."
        " Make sure you have the correct CUDA version, and that GPUs are available."
    ) from e

# ---------------------------------------------------------------------------
# Type aliases
# ---------------------------------------------------------------------------
IndexType = int | slice | Tensor | list
DeviceType = torch.device | str
LevelKind: TypeAlias = Literal["uniform", "segmented", "product"]
# Segment/graph index dtype for SegmentedLevelStorage (matches segment_lengths).
INDEX_DTYPE = torch.int32


[docs] def resolve_device(device: DeviceType | None) -> torch.device: """Return *device* with a bare ``cuda`` indexed by the current CUDA device. A storage records the device it is asked for, but ``.to("cuda")`` puts its tensors on whichever GPU is current at that moment. Recording the request verbatim makes the record disagree with the tensors as soon as the current device changes, so every later ``torch.cat`` or lazily built pointer raises a device mismatch. Resolving once at record time pins the storage to the GPU its tensors actually reached. The same resolution turns a caller's index-less ``"cuda"``, which means the device a launcher pinned this process to, into a concrete device that other placements can be compared against. Parameters ---------- device : DeviceType or None Requested device. ``None`` means ``"cpu"``. Returns ------- torch.device The requested device, with a CUDA index filled in when absent. """ resolved = torch.device(device) if device else torch.device("cpu") if resolved.type == "cuda" and resolved.index is None: return torch.device("cuda", torch.cuda.current_device()) return resolved
# --------------------------------------------------------------------------- # Dtype mapping constants # --------------------------------------------------------------------------- TORCH_DTYPE_MAP: dict[str, torch.dtype] = { "float16": torch.float16, "float32": torch.float32, "float64": torch.float64, "half": torch.float16, "float": torch.float32, "double": torch.float64, "int8": torch.int8, "int16": torch.int16, "int32": torch.int32, "int64": torch.int64, "byte": torch.int8, "short": torch.int16, "int": torch.int32, "long": torch.int64, "uint8": torch.uint8, "uint16": torch.uint16, "uint32": torch.uint32, "uint64": torch.uint64, "ubyte": torch.uint8, "complex64": torch.complex64, "complex128": torch.complex128, "cfloat": torch.complex64, "cdouble": torch.complex128, "bool": torch.bool, "bool_": torch.bool, } TORCH_DTYPE_MAP_INVERSE: dict[torch.dtype, str] = { torch.float16: "float16", torch.float32: "float32", torch.float64: "float64", torch.int8: "int8", torch.int16: "int16", torch.int32: "int32", torch.int64: "int64", torch.uint8: "uint8", torch.uint16: "uint16", torch.uint32: "uint32", torch.uint64: "uint64", torch.complex64: "complex64", torch.complex128: "complex128", torch.bool: "bool", } def effective_dtype(declared: str | torch.dtype | None) -> torch.dtype | str | None: """Resolve known dtype aliases while preserving unknown strings. Parameters ---------- declared : str, torch.dtype, or None Dtype name or object declared by the schema. Returns ------- torch.dtype, str, or None The resolved dtype for a known alias, or the original value when it is ``None``, a ``torch.dtype``, or an unknown string. """ if declared is None or isinstance(declared, torch.dtype): return declared return TORCH_DTYPE_MAP.get(declared, declared) # --------------------------------------------------------------------------- # Domain-specific defaults (aligned with nvalchemi naming conventions) # --------------------------------------------------------------------------- DEFAULT_ATTRIBUTE_MAP: dict[str, set[str]] = { "atoms": { "positions", "atomic_numbers", "forces", "velocities", "charges", "atomic_masses", }, "edges": { "neighbor_list", "edge_embeddings", "shifts", "neighbor_list_shifts", }, "system": { "cell", "pbc", "energy", "dipole", "stress", "virial", }, } DEFAULT_DTYPES: dict[str, str] = { "positions": "float32", "atomic_numbers": "int64", "forces": "float32", "velocities": "float32", "charges": "float32", "atomic_masses": "float32", "neighbor_list": "int64", "edge_embeddings": "float32", "shifts": "float32", "neighbor_list_shifts": "float32", "cell": "float32", "pbc": "bool", "energy": "float64", "dipole": "float32", "stress": "float32", "virial": "float32", } DEFAULT_SEGMENTED_GROUPS: set[str] = {"atoms", "edges"} _BUILTIN_LEVEL_ORDER = ("atoms", "edges", "system") # --------------------------------------------------------------------------- # LevelSchema # ---------------------------------------------------------------------------
[docs] class LevelSchema: """Registry mapping attributes to levels, dtypes, and level kinds. Parameters ---------- group_to_attrs : dict[str, set[str]] | None, optional Mapping from group names to sets of attribute names. Defaults to ``None`` (uses ``DEFAULT_ATTRIBUTE_MAP``). segmented_groups : set[str] | None, optional Group names whose data has variable segment lengths. Defaults to ``None`` (uses ``DEFAULT_SEGMENTED_GROUPS``). dtypes : dict[str, str] | None, optional Per-attribute dtype strings (keys of ``TORCH_DTYPE_MAP``). Defaults to ``None`` (uses ``DEFAULT_DTYPES``). Raises ------ ValueError If *dtypes* is provided but its keys don't match the attribute set. """ def __init__( self, group_to_attrs: dict[str, set[str]] | None = None, segmented_groups: set[str] | None = None, dtypes: dict[str, str] | None = None, ) -> None: if group_to_attrs is None: group_to_attrs = DEFAULT_ATTRIBUTE_MAP self.group_to_attrs: dict[str, set[str]] = { k: v.copy() for k, v in group_to_attrs.items() } self.attr_to_group: dict[str, str] = { attr: group for group, attrs in self.group_to_attrs.items() for attr in attrs } self.segmented_groups: set[str] = ( segmented_groups.copy() if segmented_groups is not None else DEFAULT_SEGMENTED_GROUPS.copy() ) if dtypes is not None: provided = set(dtypes.keys()) expected = set(self.attr_to_group.keys()) if provided != expected: raise ValueError( f"dtype keys must match attribute set. Missing: {expected - provided}, " f"extra: {provided - expected}" ) self.dtypes: dict[str, str] = dtypes.copy() else: self.dtypes = DEFAULT_DTYPES.copy() self.level_kinds: dict[str, LevelKind] = { group: "segmented" if group in self.segmented_groups else "uniform" for group in self.group_to_attrs } self.product_parents: dict[str, tuple[str, str]] = {} builtin_levels = [ group for group in _BUILTIN_LEVEL_ORDER if group in self.group_to_attrs ] custom_levels = [ group for group in self.group_to_attrs if group not in _BUILTIN_LEVEL_ORDER ] self._level_names = [*builtin_levels, *custom_levels] # -- Mutation ----------------------------------------------------------- @staticmethod def _validate_level_name(name: str, parameter: str) -> None: if not isinstance(name, str): raise TypeError(f"{parameter} must be a string") if not name.strip(): raise ValueError(f"{parameter} must not be empty or whitespace") def _validate_can_make_uniform(self, name: str) -> None: dependents = [ product for product, parents in self.product_parents.items() if name in parents ] if dependents: raise ValueError( f"Level '{name}' cannot be made uniform because it is a parent of " f"product level(s): {', '.join(dependents)}" )
[docs] def add_level(self, name: str, *, segmented: bool) -> None: """Register an ordinary uniform or segmented level. Repeating the same definition is an idempotent operation. Parameters ---------- name : str Name of the level. segmented : bool Whether the level has variable per-system cardinality. Raises ------ TypeError If *name* is not a string or *segmented* is not a Boolean. ValueError If *name* is empty or already has a different kind. """ self._validate_level_name(name, "name") if not isinstance(segmented, bool): raise TypeError("segmented must be a bool") kind: LevelKind = "segmented" if segmented else "uniform" existing_kind = self.level_kinds.get(name) if existing_kind is not None: if existing_kind == kind: return raise ValueError( f"Level '{name}' is already registered as {existing_kind}, not {kind}" ) self.group_to_attrs.setdefault(name, set()) self.level_kinds[name] = kind self._level_names.append(name) if segmented: self.segmented_groups.add(name) else: self.segmented_groups.discard(name)
[docs] def add_product_level(self, name: str, *, left: str, right: str) -> None: """Register a segmented Cartesian product of two base levels. Product levels preserve parent order and may use the same segmented parent on both axes. Repeating an identical definition is idempotent. Parameters ---------- name : str Name of the product level. left : str Registered segmented level for the left entity axis. right : str Registered segmented level for the right entity axis. Raises ------ TypeError If any level name is not a string. KeyError If either parent is not registered. ValueError If a name is empty, a parent is not a segmented base level, or the requested definition conflicts with an existing level. """ self._validate_level_name(name, "name") self._validate_level_name(left, "left") self._validate_level_name(right, "right") existing_kind = self.level_kinds.get(name) if existing_kind == "product": if self.product_parents[name] == (left, right): return raise ValueError( f"Product level '{name}' is already registered with parents " f"{self.product_parents[name]}, not {(left, right)}" ) if existing_kind is not None: raise ValueError( f"Level '{name}' is already registered as {existing_kind}, not product" ) if name == left or name == right: raise ValueError(f"Product level '{name}' cannot reference itself") for parent_name, axis in ((left, "left"), (right, "right")): if parent_name not in self.level_kinds: raise KeyError(f"{axis} parent level '{parent_name}' is not registered") parent_kind = self.level_kinds[parent_name] if parent_kind != "segmented": raise ValueError( f"{axis} parent level '{parent_name}' must be a segmented " f"base level, not {parent_kind}" ) self.group_to_attrs.setdefault(name, set()) self.level_kinds[name] = "product" self.product_parents[name] = (left, right) self._level_names.append(name) self.segmented_groups.add(name)
[docs] def set( self, attr_name: str, group_name: str, dtype: str | torch.dtype | None = None, is_segmented: bool | None = None, ) -> None: """Register or update an attribute's group, dtype, and segmentation. Parameters ---------- attr_name : str Name of the attribute. group_name : str Target group for the attribute. dtype : str or torch.dtype, optional Data type for the attribute. is_segmented : bool, optional Whether *group_name* should be marked as segmented. Raises ------ ValueError If the requested segmentation would invalidate a product level or *dtype* is an unsupported ``torch.dtype`` value. """ normalized_dtype = dtype if isinstance(dtype, torch.dtype): try: normalized_dtype = TORCH_DTYPE_MAP_INVERSE[dtype] except KeyError as exc: raise ValueError(f"Unsupported torch dtype: {dtype}") from exc level_kind = self.level_kinds.get(group_name) if level_kind == "product" and is_segmented is False: raise ValueError(f"Product level '{group_name}' cannot be made uniform") if level_kind == "segmented" and is_segmented is False: self._validate_can_make_uniform(group_name) existing_group = self.attr_to_group.get(attr_name) if existing_group is not None and existing_group != group_name: self.group_to_attrs[existing_group].discard(attr_name) self.attr_to_group[attr_name] = group_name if group_name not in self.group_to_attrs: self.group_to_attrs[group_name] = set() self.group_to_attrs[group_name].add(attr_name) if level_kind is None: if is_segmented is None: is_segmented = group_name in self.segmented_groups level_kind = "segmented" if is_segmented else "uniform" self.level_kinds[group_name] = level_kind self._level_names.append(group_name) if level_kind == "product": self.segmented_groups.add(group_name) elif is_segmented is not None: if is_segmented: self.segmented_groups.add(group_name) self.level_kinds[group_name] = "segmented" else: self.segmented_groups.discard(group_name) self.level_kinds[group_name] = "uniform" if normalized_dtype is not None: self.dtypes[attr_name] = normalized_dtype
[docs] def mark_group_segmented(self, group_name: str) -> None: """Mark *group_name* as segmented.""" self.segmented_groups.add(group_name) if group_name in self.level_kinds and self.level_kinds[group_name] != "product": self.level_kinds[group_name] = "segmented"
[docs] def unmark_group_segmented(self, group_name: str) -> None: """Remove the segmented flag from *group_name*. Raises ------ KeyError If *group_name* is not currently segmented. ValueError If *group_name* is a product or a parent of a product. """ if self.level_kinds.get(group_name) == "product": raise ValueError(f"Product level '{group_name}' cannot be made uniform") self._validate_can_make_uniform(group_name) self.segmented_groups.remove(group_name) if group_name in self.level_kinds: self.level_kinds[group_name] = "uniform"
def _merged_with(self, extension: LevelSchema) -> LevelSchema: """Return an additive merge without modifying either input schema. The existing schema remains authoritative for declarations it already owns. New levels and fields are appended from *extension* in its definition order. Equivalent dtype aliases compare through their effective :class:`torch.dtype` values. Parameters ---------- extension : LevelSchema Schema declarations to add. Returns ------- LevelSchema Independent merged schema. Raises ------ TypeError If *extension* is not a :class:`LevelSchema`. ValueError If an existing level, product, field owner, or dtype conflicts. """ if not isinstance(extension, LevelSchema): raise TypeError( f"extension must be a LevelSchema, got {type(extension).__name__}" ) merged = self.clone() for level_name in extension.level_names: kind = extension.level_kind(level_name) if kind == "product": left, right = extension.product_parents[level_name] merged.add_product_level(level_name, left=left, right=right) else: merged.add_level(level_name, segmented=kind == "segmented") for attr_name, group_name in extension.attr_to_group.items(): existing_group = merged.attr_to_group.get(attr_name) if existing_group is not None and existing_group != group_name: raise ValueError( f"Field '{attr_name}' is already assigned to level " f"'{existing_group}', not '{group_name}'" ) existing_dtype = merged.dtypes.get(attr_name) extension_dtype = extension.dtypes.get(attr_name) if existing_dtype is not None and extension_dtype is not None: existing_effective = effective_dtype(existing_dtype) extension_effective = effective_dtype(extension_dtype) if existing_effective != extension_effective: raise ValueError( f"Field '{attr_name}' has incompatible declared dtypes: " f"{existing_dtype} vs {extension_dtype}" ) if existing_group is None or ( existing_dtype is None and extension_dtype is not None ): merged.set( attr_name, group_name, dtype=extension_dtype, is_segmented=merged.level_kind(group_name) != "uniform", ) return merged # -- Queries ------------------------------------------------------------
[docs] def is_segmented_group(self, group_name: str) -> bool: """Return whether *group_name* is segmented.""" return group_name in self.segmented_groups
[docs] def is_segmented_attr(self, attr_name: str) -> bool: """Return whether *attr_name* belongs to a segmented group. Raises ------ KeyError If *attr_name* is not registered. """ if attr_name not in self.attr_to_group: raise KeyError(f"Attribute '{attr_name}' not found") return self.attr_to_group[attr_name] in self.segmented_groups
@property def level_names(self) -> tuple[str, ...]: """Return registered level names in deterministic definition order.""" return tuple(self._level_names)
[docs] def level_kind(self, name: str) -> LevelKind: """Return the kind of a registered level. Parameters ---------- name : str Name of the level. Returns ------- LevelKind ``"uniform"``, ``"segmented"``, or ``"product"``. Raises ------ KeyError If *name* is not a registered level. """ if name not in self.level_kinds: raise KeyError(f"Level '{name}' not found") return self.level_kinds[name]
[docs] def group(self, attr_name: str) -> str: """Return the group name for *attr_name*. Raises ------ KeyError If *attr_name* is not registered. """ if attr_name not in self.attr_to_group: raise KeyError(f"Attribute '{attr_name}' not found") return self.attr_to_group[attr_name]
[docs] def dtype(self, attr_name: str) -> str: """Return the dtype string for *attr_name*. Raises ------ KeyError If *attr_name* has no registered dtype. """ if attr_name not in self.dtypes: raise KeyError(f"Attribute '{attr_name}' not found in dtype registry") return self.dtypes[attr_name]
# -- Copy ---------------------------------------------------------------
[docs] def clone(self) -> LevelSchema: """Return an independent deep copy.""" cloned = LevelSchema.__new__(LevelSchema) cloned.group_to_attrs = {g: a.copy() for g, a in self.group_to_attrs.items()} cloned.attr_to_group = self.attr_to_group.copy() cloned.segmented_groups = self.segmented_groups.copy() cloned.dtypes = self.dtypes.copy() cloned.level_kinds = self.level_kinds.copy() cloned.product_parents = self.product_parents.copy() cloned._level_names = self._level_names.copy() return cloned
# --------------------------------------------------------------------------- # warp-accelerated helper # --------------------------------------------------------------------------- _INT32_MAX: int = 2**31 - 1 def _validate_buffer_field_pairs( destination: TensorDict, source: TensorDict, ) -> list[str]: """Validate participating buffer fields and return them in destination order.""" fields = [key for key in destination.keys() if key in source.keys()] for key in fields: dest_dtype = destination[key].dtype source_dtype = source[key].dtype if dest_dtype != source_dtype: raise ValueError( f"Field '{key}' has incompatible dtypes: {dest_dtype} vs {source_dtype}" ) if dest_dtype not in TORCH_TO_WP: raise ValueError( f"Field '{key}' dtype {dest_dtype} is not supported by buffer kernels" ) return fields def _validate_buffer_field_dtypes(data: TensorDict) -> list[str]: """Validate buffer field dtypes and return fields in storage order.""" fields = list(data.keys()) for key in fields: dtype = data[key].dtype if dtype not in TORCH_TO_WP: raise ValueError( f"Field '{key}' dtype {dtype} is not supported by buffer kernels" ) return fields def _checked_segment_metadata( segment_lengths: list[int] | Tensor, device: DeviceType, *, batch_ptr: Tensor | None = None, batch_ptr_capacity: int | None = None, check: bool = True, ) -> tuple[Tensor, Tensor]: """Normalize segmented lengths and build an int32 pointer without wrapping. All arithmetic is performed in int64 before the in-memory int32 metadata is materialized. A supplied pointer is retained (including its spare capacity) after the same range check; otherwise a logical pointer is built from the supplied lengths and optionally expanded to *batch_ptr_capacity*. With ``check=False`` the range check, which reads a flag back to the host, is skipped; use it only for metadata that is already known to be in range. """ lengths64 = to_tensor(segment_lengths, device=device).to(torch.int64) if lengths64.ndim != 1: raise ValueError("segment_lengths must be one-dimensional") cumulative64 = torch.cumsum(lengths64, dim=0, dtype=torch.int64) supplied64: Tensor | None = None if batch_ptr is not None: supplied64 = to_tensor(batch_ptr, device=device).to(torch.int64) if supplied64.ndim != 1: raise ValueError("batch_ptr must be one-dimensional") if check: invalid_metadata = ( (lengths64 < 0) | (lengths64 > _INT32_MAX) | (cumulative64 > _INT32_MAX) ).any() if supplied64 is not None: invalid_metadata = ( invalid_metadata | ((supplied64 < 0) | (supplied64 > _INT32_MAX)).any() ) if check and bool(invalid_metadata): if bool((lengths64 < 0).any()): raise ValueError(f"Segment lengths cannot be negative: {lengths64}") if bool((lengths64 > _INT32_MAX).any()): raise OverflowError( f"Segment length exceeds signed int32 maximum ({_INT32_MAX})" ) if bool((cumulative64 > _INT32_MAX).any()): raise OverflowError( f"Segment pointer exceeds signed int32 maximum ({_INT32_MAX})" ) if supplied64 is not None and bool((supplied64 < 0).any()): raise ValueError(f"batch_ptr values cannot be negative: {supplied64}") raise OverflowError( f"Supplied batch_ptr exceeds signed int32 maximum ({_INT32_MAX})" ) logical_ptr = torch.cat( [ torch.zeros(1, device=device, dtype=torch.int32), cumulative64.to(torch.int32), ] ) if supplied64 is not None: return lengths64.to(torch.int32), supplied64.to(torch.int32) capacity = max(batch_ptr_capacity or 0, logical_ptr.numel()) if capacity == logical_ptr.numel(): return lengths64.to(torch.int32), logical_ptr ptr = torch.empty(capacity, device=device, dtype=torch.int32) ptr[: logical_ptr.numel()] = logical_ptr ptr[logical_ptr.numel() :] = logical_ptr[-1] return lengths64.to(torch.int32), ptr @wp.kernel(enable_backward=False) def _expand_segments_kernel( starts: wp.array(dtype=Any), lengths: wp.array(dtype=Any), offsets: wp.array(dtype=Any), output: wp.array(dtype=Any), ) -> None: """Write ``range(starts[tid], starts[tid]+lengths[tid])`` at ``offsets[tid]``. Parameters ---------- starts : wp.array Per-segment start element index. lengths : wp.array Per-segment element count. offsets : wp.array Per-segment write offset into *output*. output : wp.array Pre-allocated 1-D buffer receiving the expanded element indices. """ tid = wp.tid() s = starts[tid] n = lengths[tid] o = offsets[tid] for _i in range(wp.int32(n)): i = type(s)(_i) output[o + i] = s + i _WP_INT_TYPES = [wp.int32, wp.int64] _expand_segments_overloads: dict[type, Any] = {} for _wp_t in _WP_INT_TYPES: _expand_segments_overloads[_wp_t] = wp.overload( _expand_segments_kernel, [ wp.array(dtype=_wp_t), wp.array(dtype=_wp_t), wp.array(dtype=_wp_t), wp.array(dtype=_wp_t), ], ) _TORCH_TO_WP: dict[torch.dtype, type] = { torch.int32: wp.int32, torch.int64: wp.int64, } def _expand_segments_warp( seg_idx: torch.Tensor, batch_ptr: torch.Tensor, index_dtype: torch.dtype, ) -> torch.Tensor: """Expand segment indices to element indices using a Warp kernel. The kernel overload matching *index_dtype* is selected automatically. Gathered pointer values are cast to that dtype before launch so Warp sees arguments matching the selected overload. The launch and the output follow *batch_ptr*'s own device rather than a requested one, because a storage may record an index-less ``cuda`` while its tensors sit on a non-default GPU; launching such a storage against the current device reads unmapped memory. The current device is restored afterwards, since a Warp launch leaves its own device selected. Parameters ---------- seg_idx : torch.Tensor 1-D tensor of selected segment indices, on *batch_ptr*'s device. batch_ptr : torch.Tensor Cumulative segment pointer of length ``num_segments + 1``. index_dtype : torch.dtype Integer dtype (``torch.int32`` or ``torch.int64``) for the output tensor and kernel selection. Returns ------- torch.Tensor 1-D tensor of element-level indices on *batch_ptr*'s device. Raises ------ ValueError If *index_dtype* is not ``torch.int32`` or ``torch.int64``. """ wp_dtype = _TORCH_TO_WP.get(index_dtype) if wp_dtype is None: raise ValueError( f"Unsupported index_dtype {index_dtype}; " f"expected torch.int32 or torch.int64" ) kernel = _expand_segments_overloads[wp_dtype] if seg_idx.numel() == 0: return torch.empty(0, device=batch_ptr.device, dtype=index_dtype) starts = batch_ptr[seg_idx].to(index_dtype) ends = batch_ptr[seg_idx + 1].to(index_dtype) lengths = ends - starts if bool(((starts < 0) | (ends < 0)).any()): raise ValueError("batch_ptr values cannot be negative") if bool((lengths < 0).any()): raise ValueError("Segment lengths cannot be negative") cumlen64 = torch.cumsum(lengths, dim=0, dtype=torch.int64) total = int(cumlen64[-1].item()) if total > _INT32_MAX: raise OverflowError( f"Total element count {total} exceeds int32 maximum " f"({_INT32_MAX}); the Warp kernel uses int32 loop bounds" ) # An index-less ``cuda`` device resolves to whichever device is current, # which need not be where the pointer lives. launch_device = starts.device if total == 0: return torch.empty(0, device=launch_device, dtype=index_dtype) offsets = (cumlen64 - lengths.to(torch.int64)).to(index_dtype) output = torch.empty(total, device=launch_device, dtype=index_dtype) if launch_device.type == "cuda": wp_device = f"cuda:{launch_device.index}" device_scope = torch.cuda.device(launch_device) else: wp_device = "cpu" device_scope = contextlib.nullcontext() # wp.launch leaves its own device current; restore torch's. with device_scope: wp.launch( kernel=kernel, dim=seg_idx.numel(), inputs=[ wp.from_torch(starts, dtype=wp_dtype), wp.from_torch(lengths, dtype=wp_dtype), wp.from_torch(offsets, dtype=wp_dtype), wp.from_torch(output, dtype=wp_dtype), ], device=wp_device, ) return output # --------------------------------------------------------------------------- # Tensor utility helpers # --------------------------------------------------------------------------- def to_tensor( value: Any, device: DeviceType | None = None, dtype: str | None = None, ) -> Tensor: """Convert an arbitrary value to a :class:`torch.Tensor`. Accepts tensors, numpy arrays, lists, scalars, and anything that :func:`torch.as_tensor` supports. Parameters ---------- value : Any Value to convert. device : DeviceType, optional Target device. dtype : str, optional Dtype string (looked up in ``TORCH_DTYPE_MAP``). Returns ------- Tensor Raises ------ TypeError If *value* cannot be converted to a tensor. """ if isinstance(value, list) and len(value) > 0 and isinstance(value[0], np.ndarray): value = np.array(value) # Only apply the schema dtype for non-tensor inputs (lists, numpy arrays, # scalars) where there is no existing dtype to preserve. When the input # is already a Tensor, respect its dtype — the user chose it deliberately. if dtype and not isinstance(value, Tensor): torch_dtype = TORCH_DTYPE_MAP[dtype] else: torch_dtype = None return torch.as_tensor(value, device=device, dtype=torch_dtype) def _clone_tensor(tensor: Tensor) -> Tensor: """Return a detached clone of *tensor*.""" return tensor.clone() def _concatenate_tensors(t1: Tensor, t2: Tensor) -> Tensor: """Concatenate two tensors along dim 0, coercing *t2* to *t1*'s device/dtype. Parameters ---------- t1, t2 : Tensor Tensors with matching shapes except for the first dimension. Returns ------- Tensor Raises ------ ValueError If trailing shapes don't match. """ if t1.shape[1:] != t2.shape[1:]: raise ValueError( f"Shape mismatch for concatenation: {t1.shape[1:]} vs {t2.shape[1:]}" ) return torch.cat([t1, t2.to(device=t1.device, dtype=t1.dtype)], dim=0) def _validate_trailing_shapes(tensors: list[Tensor]) -> bool: """Return ``True`` if all tensors share the same shape after dim 0.""" if not tensors: return True trailing = tensors[0].shape[1:] return all(t.shape[1:] == trailing for t in tensors) # --------------------------------------------------------------------------- # BaseLevelStorage (ABC) # --------------------------------------------------------------------------- class BaseLevelStorage(ABC): """Abstract base for dict-like tensor containers. Subclasses must implement ``__len__``, ``select``, ``update_at``, ``concatenate``, ``is_segmented``, and ``_validate_setitem``. Parameters ---------- data : dict[str, Tensor], optional Initial attribute tensors keyed by name. device : DeviceType | None, optional Target device. Defaults to ``None`` (uses ``"cpu"``). attr_map : LevelSchema, optional Shared attribute registry. validate : bool Whether to run shape checks on every ``__setitem__``. """ def __init__( self, data: dict[str, Tensor] | TensorDict | None = None, device: DeviceType | None = None, attr_map: LevelSchema | None = None, validate: bool = True, ) -> None: self._attr_map = attr_map if attr_map else LevelSchema() self.device = resolve_device(device) self.validate = validate if data is None: self._data = TensorDict({}, batch_size=torch.Size([0]), device=self.device) elif isinstance(data, TensorDict): self._data = data else: if not isinstance(data, dict): raise TypeError( f"data must be a dict or TensorDict, got {type(data).__name__}" ) for key in data: if not isinstance(key, str): raise TypeError( f"Attribute keys must be strings, got {type(key).__name__}" ) if not data: self._data = TensorDict( {}, batch_size=torch.Size([0]), device=self.device ) else: converted = { k: to_tensor( v, device=self.device, dtype=self._attr_map.dtypes.get(k), ) for k, v in data.items() } first_dim = next(iter(converted.values())).shape[0] self._data = TensorDict( converted, batch_size=[first_dim], device=self.device, ) # -- Properties --------------------------------------------------------- @property def attr_map(self) -> LevelSchema: """The :class:`LevelSchema` associated with this container.""" return self._attr_map @attr_map.setter def attr_map(self, attr_map: LevelSchema) -> None: self._attr_map = attr_map @property def num_systems(self) -> int: """Number of systems (same as ``len(self)``).""" return len(self) @property def num_atoms(self) -> int: """Total number of atoms across all systems. For segmented containers this is the first dimension of any atoms-group attribute. For uniform containers it is ``dim0 * dim1``. Raises ------ ValueError If no atoms-group attribute is present. """ if self._data.is_empty(): return 0 atoms_attrs = self._attr_map.group_to_attrs.get("atoms", set()) for attr_name, value in self._data.items(): if attr_name in atoms_attrs: if self.is_segmented(): return value.shape[0] return value.shape[0] * value.shape[1] raise ValueError("Cannot determine num_atoms: no atoms-group attributes found") # -- Abstract ----------------------------------------------------------- @abstractmethod def __len__(self) -> int: ... @abstractmethod def select(self, idx: IndexType, *, drop: Collection[str] = ()) -> BaseLevelStorage: """Return a container of only the selected samples, without *drop* keys.""" @abstractmethod def update_at(self, key: str, value: Tensor, idx: IndexType) -> None: """In-place update of *key* at the given indices.""" @abstractmethod def concatenate( self, other: BaseLevelStorage | dict[str, Tensor], strict: bool = False, ) -> BaseLevelStorage: """Concatenate *other* into this container (in-place).""" @abstractmethod def is_segmented(self) -> bool: """Return ``True`` if this container uses segmented storage.""" @abstractmethod def _validate_setitem(self, key: str, value: Tensor) -> None: ... # -- Dict-like interface ------------------------------------------------ def __getitem__(self, key: str | IndexType) -> Tensor | BaseLevelStorage: """Retrieve an attribute by name or select a subset by index.""" if isinstance(key, str): if key not in self._data.keys(): raise KeyError(f"Key '{key}' not found") return self._data[key] if isinstance(key, (slice, int, Tensor, list)): return self.select(key) raise TypeError(f"Invalid index type: {type(key).__name__}") def __setitem__(self, key: str, value: Any) -> None: """Set an attribute, converting *value* to a tensor.""" dtype = self._attr_map.dtypes.get(key, None) tensor = to_tensor(value, device=self.device, dtype=dtype) if self.validate: self._validate_setitem(key, tensor) self._data[key] = tensor def __contains__(self, key: str) -> bool: return key in self._data.keys() def __iter__(self) -> Iterator[str]: return iter(self._data.keys()) def __delitem__(self, key: str) -> None: if key not in self._data.keys(): raise KeyError(f"Key '{key}' not found") del self._data[key] def keys(self) -> Iterator[str]: """Iterate over attribute names.""" return iter(self._data.keys()) def values(self) -> Iterator[Tensor]: """Iterate over attribute tensors.""" return self._data.values() def items(self) -> Iterator[tuple[str, Tensor]]: """Iterate over ``(name, tensor)`` pairs.""" return self._data.items() def get(self, key: str, default: Tensor | None = None) -> Tensor | None: """Return the tensor for *key*, or *default* if absent.""" return self._data.get(key, default) def pop(self, key: str, default: Tensor | None = None) -> Tensor: """Remove and return the tensor for *key*. Parameters ---------- key : str Attribute name. default : Tensor, optional Fallback value if *key* is missing. Raises ------ KeyError If *key* is absent and no *default* was given. """ if key not in self._data.keys(): if default is not None: return default raise KeyError(f"Key '{key}' not found") value = self._data[key].clone() del self._data[key] return value # -- Copy / device ------------------------------------------------------ def deepcopy(self) -> BaseLevelStorage: """Return a new container with cloned tensors (attr_map is shared).""" cloned = {k: _clone_tensor(v) for k, v in self._data.items()} return self.__class__(cloned, device=self.device, validate=False) def copy(self) -> BaseLevelStorage: """Return a shallow copy (tensors are **not** cloned).""" return self.__class__(self._data, device=self.device, validate=False) def clone(self, *, drop: Collection[str] = ()) -> BaseLevelStorage: """Return a deep copy including an independent attr_map clone. Parameters ---------- drop : Collection[str], optional Keys left out of the copy instead of being cloned. Default ``()``. """ cloned = {k: _clone_tensor(v) for k, v in self._data.items() if k not in drop} return self.__class__( cloned, device=self.device, attr_map=self._attr_map.clone(), validate=False, ) def to_device( self, device: DeviceType, *, non_blocking: bool = False ) -> BaseLevelStorage: """Move all tensors to *device* (in-place). Parameters ---------- device : DeviceType Target device. A bare ``"cuda"`` is recorded as the CUDA device current at the time of the move, which is where the tensors land. non_blocking : bool, default False Whether tensor copies may be asynchronous when supported. Returns ------- Self For method chaining. """ device = resolve_device(device) self.device = device self._data = self._data.to(device, non_blocking=non_blocking) return self # --------------------------------------------------------------------------- # UniformLevelStorage (uniform / dense) # --------------------------------------------------------------------------- class UniformLevelStorage(BaseLevelStorage): """Container for uniform data where every attribute shares the same first dimension. Parameters ---------- data : dict[str, Tensor], optional Initial attribute tensors. device : DeviceType, optional Target device. attr_map : LevelSchema, optional Shared attribute registry. validate : bool If ``True``, ensure all attributes have the same ``shape[0]``. Raises ------ ValueError If validation is enabled and first-dimension sizes differ. """ def __init__( self, data: dict[str, Tensor] | None = None, device: DeviceType | None = None, attr_map: LevelSchema | None = None, validate: bool = True, ) -> None: # Validate first dims before building TensorDict (TensorDict requires consistent batch) if data is not None and isinstance(data, dict) and validate and len(data) > 1: first_dim = next(iter(data.values())).shape[0] for key, value in data.items(): if value.shape[0] != first_dim: raise ValueError( f"Inconsistent first dimension: expected {first_dim}, " f"got {value.shape[0]} for '{key}'" ) super().__init__(data, device, attr_map, validate) if data is not None and validate and not self._data.is_empty(): first_key = next(iter(self._data.keys())) batch_size = self._data[first_key].shape[0] for key in self._data.keys(): if self._data[key].shape[0] != batch_size: raise ValueError( f"Inconsistent first dimension between '{first_key}' and " f"'{key}': {batch_size} vs {self._data[key].shape[0]}" ) def __len__(self) -> int: n = getattr(self, "_num_kept", None) if n is not None: return n if self._data.is_empty(): # batch_size is set to [N] when constructed from a sized TensorDict # (e.g. the empty system group created by from_data_list). bs = self._data.batch_size return int(bs[0]) if bs else 0 return self._data.shape[0] def num_elements(self) -> int: """Total number of elements (rows); after defrag, same as len(self).""" return len(self) def _validate_setitem(self, key: str, value: Tensor) -> None: if not self._data.is_empty() and len(value) != len(self): raise ValueError(f"Length mismatch: {len(value)} vs {len(self)}") def select( self, idx: IndexType, *, drop: Collection[str] = () ) -> UniformLevelStorage: """Select a subset of samples by index. Parameters ---------- idx : int, slice, or Tensor Index specification. drop : Collection[str], optional Keys left out of the selection instead of being copied. Default ``()``. Returns ------- UniformLevelStorage """ if isinstance(idx, int): idx = slice(idx, idx + 1) elif not isinstance(idx, slice): idx = self._prepare_index(idx) data = self._data.exclude(*drop) if drop else self._data selected_td = data[idx] return self.__class__(selected_td, device=self.device, validate=False) def _prepare_index(self, idx: Any) -> Tensor: """Coerce *idx* to a tensor on ``self.device``, preserving integer dtype.""" idx = to_tensor(idx) return idx.to(device=self.device) def update_at(self, key: str, value: Any, idx: IndexType) -> None: """Update attribute *key* at the given indices. Parameters ---------- key : str Attribute name. value : Any Replacement values (converted to tensor). idx : IndexType Target indices. Raises ------ KeyError If *key* is not present. ValueError If *idx* is not 1-D when given as a tensor. """ if key not in self._data.keys(): raise KeyError(f"Key '{key}' not found") value = to_tensor(value, self.device) if isinstance(idx, (slice, int)): self._data.set_at_(key, value, index=idx) else: idx = to_tensor(idx) if idx.ndim != 1: raise ValueError(f"Index must be 1-D, got {idx.ndim}-D") self._data.set_at_(key, value, index=idx.to(device=self.device)) def concatenate( self, other: UniformLevelStorage | dict[str, Tensor], strict: bool = False, ) -> UniformLevelStorage: """Concatenate *other* along dimension 0 (in-place). Parameters ---------- other : UniformLevelStorage or dict[str, Tensor] Data to append. strict : bool If ``True``, require identical attribute sets. Returns ------- Self For method chaining. Raises ------ ValueError On key mismatch (strict) or incompatible trailing shapes. """ self_keys = set(self.keys()) other_keys = set(other.keys()) common = self_keys & other_keys if strict and self_keys != other_keys: raise ValueError(f"Keys mismatch: {self_keys} vs {other_keys}") if not common: # No keys to concatenate, but extend with zeros so that batch_size # stays aligned (matches the behavior of extend_for_appended_graphs). n_other = len(other) return self.extend_for_appended_graphs(n_other) # TensorDict enforces batch_size; replace with new TensorDict of concatenated data new_data = {} for key in common: other_tensor = to_tensor(other[key], self.device) if not _validate_trailing_shapes([self[key], other_tensor]): raise ValueError( f"Key '{key}' has incompatible shape: " f"{self[key].shape} vs {other_tensor.shape}" ) new_data[key] = _concatenate_tensors(self._data[key], other_tensor) new_len = next(iter(new_data.values())).shape[0] self._data = TensorDict( new_data, batch_size=[new_len], device=self.device, ) if hasattr(self, "_num_kept"): object.__delattr__(self, "_num_kept") return self def extend_for_appended_graphs(self, n: int) -> UniformLevelStorage: """Extend all tensors by *n* rows of zeros (for batches that lack this group). Used when appending a batch that does not have this group: the first dimension (num graphs) is extended so segment counts stay aligned. Parameters ---------- n : int Number of graph slots to add (zero-padded). Returns ------- Self For method chaining. """ if n <= 0: return self if self._data.is_empty(): return self new_data = {} for key, tensor in self._data.items(): pad = torch.zeros( n, *tensor.shape[1:], dtype=tensor.dtype, device=tensor.device ) new_data[key] = torch.cat([tensor, pad], dim=0) new_len = next(iter(new_data.values())).shape[0] self._data = TensorDict( new_data, batch_size=[new_len], device=self.device, ) if hasattr(self, "_num_kept"): object.__delattr__(self, "_num_kept") return self def compute_put_per_system_fit_mask( self, source: UniformLevelStorage, source_mask: Tensor, fit_mask: Tensor, ) -> None: """Compute which source rows would fit in this storage; write result into fit_mask in place. Parameters ---------- source : UniformLevelStorage Source storage; length gives num_systems. source_mask : Tensor, shape (num_systems,), dtype bool True for each system considered for copy. fit_mask : Tensor, shape (num_systems,), dtype bool Output written in place: True where that source row would fit. Notes ----- No data is copied. Use with put after combining (e.g. logical_and) with other levels' fit masks so only systems that fit in every level are copied. The first ``len(self)`` slots are treated as occupied, so a storage without a fill count (``_num_kept``) counts as full and nothing fits. """ n_src = len(source) if source_mask.shape[0] != n_src: raise ValueError( f"source_mask shape {source_mask.shape[0]} != len(source) {n_src}" ) if fit_mask.shape[0] != n_src: raise ValueError(f"fit_mask shape {fit_mask.shape[0]} != {n_src}") source_mask = source_mask.to(device=self.device, dtype=torch.bool) fit_mask = fit_mask.to(device=self.device, dtype=torch.bool) occupied = torch.arange(self._data.shape[0], device=self.device) < len(self) compute_put_fit_mask_per_system(source_mask, occupied, fit_mask) def put( self, src: UniformLevelStorage, mask: Tensor, *, copied_mask: Tensor | None = None, num_selected: int | None = None, ) -> None: """Put rows where mask[i] is True from src into this storage (buffer). Supported payload dtypes are bool, float32, float64, int32, and int64; source and destination dtypes must match. Rows are appended after the first ``len(self)`` slots, and only as many rows as fit in the remaining slots are copied. ``len(self)`` is the fill count (``_num_kept``) of a pre-allocated buffer. A storage without one counts as full, so nothing is copied. Uses Warp buffer kernels. If copied_mask is provided, it is updated in place with True for each row that was copied. Parameters ---------- src : UniformLevelStorage Source storage; same attribute keys as self. mask : Tensor (num_systems,) bool, True = copy this row. copied_mask : Tensor, optional (num_systems,) bool; if provided, modified in place with which rows were actually copied. If None, stored as ``_copied_mask`` for use by :meth:`defrag`. num_selected : int, optional Number of True entries in ``mask``. Supply only when all selected rows are known to fit. Avoids a host sync to count copied rows. """ if self._data.is_empty() or src._data.is_empty(): raise ValueError("put requires non-empty source and dest") fields = _validate_buffer_field_pairs(self._data, src._data) if not fields: raise ValueError("put requires at least one common attribute") n_src = len(src) dest_capacity = self._data.shape[0] if mask.shape[0] != n_src: raise ValueError(f"mask shape {mask.shape[0]} != len(src) {n_src}") mask = mask.to(device=self.device, dtype=torch.bool) for key in fields: dest_t = self._data[key] if dest_t.shape[0] < dest_capacity: raise ValueError( f"dest attribute '{key}' first dim {dest_t.shape[0]} < {dest_capacity}" ) if copied_mask is not None: if copied_mask.shape[0] != n_src: raise ValueError(f"copied_mask shape {copied_mask.shape[0]} != {n_src}") out_mask = copied_mask.to(device=self.device, dtype=torch.bool) else: out_mask = torch.zeros(n_src, device=self.device, dtype=torch.bool) object.__setattr__(src, "_copied_mask", out_mask) initial_dest_mask = torch.arange(dest_capacity, device=self.device) < len(self) first_key, *remaining_keys = fields first_dest_mask = initial_dest_mask.clone() put_masked_per_system( src._data[first_key], mask, self._data[first_key], first_dest_mask, out_mask, ) for key in remaining_keys: src_t = src._data[key] dest_t = self._data[key] field_dest_mask = initial_dest_mask.clone() field_copied_mask = out_mask.clone() put_masked_per_system( src_t, mask, dest_t, field_dest_mask, field_copied_mask, ) num_copied = num_selected if num_selected is not None else out_mask.sum().item() if num_copied > 0 and getattr(self, "_num_kept", None) is not None: object.__setattr__(self, "_num_kept", self._num_kept + num_copied) def defrag( self, copied_mask: Tensor | None = None, ) -> UniformLevelStorage: """Defrag in-place: remove rows where copied_mask[i] is True. Rows with copied_mask[i] True (previously put) are dropped; remaining rows move to the front in place. Payloads must use a dtype supported by the buffer kernels (``bool``, ``float32``, ``float64``, ``int32``, or ``int64``). All fields are validated before compaction. Buffer shape is unchanged for fixed-size batches. Parameters ---------- copied_mask : Tensor, optional (num_systems,) bool; if None, uses ``_copied_mask`` from the last :meth:`put`. Returns ------- Self For method chaining. """ if self._data.is_empty(): return self n_src = len(self) if copied_mask is None: copied_mask = getattr(self, "_copied_mask", None) if copied_mask is None: raise ValueError("defrag requires copied_mask or a prior put") else: copied_mask = copied_mask.to(device=self.device, dtype=torch.bool) if copied_mask.shape[0] != n_src: raise ValueError(f"copied_mask shape {copied_mask.shape[0]} != {n_src}") fields = _validate_buffer_field_dtypes(self._data) for key in fields: t = self._data[key] # Pass a clone so the kernel's in-place mask update doesn't affect other keys defrag_per_system(t, copied_mask.clone()) object.__setattr__(self, "_num_kept", int((~copied_mask).sum().item())) if hasattr(self, "_copied_mask"): object.__delattr__(self, "_copied_mask") return self def is_segmented(self) -> bool: return False def to_device( self, device: DeviceType, *, non_blocking: bool = False ) -> UniformLevelStorage: """Move all tensors to *device* (in-place).""" super().to_device(device, non_blocking=non_blocking) return self def clone(self, *, drop: Collection[str] = ()) -> UniformLevelStorage: """Return a deep copy minus *drop*; copies _num_kept if set (e.g. after defrag).""" out = super().clone(drop=drop) if getattr(self, "_num_kept", None) is not None: object.__setattr__(out, "_num_kept", self._num_kept) return out def __repr__(self) -> str: attrs = ", ".join(f"'{k}': {self._data[k].shape}" for k in self._data.keys()) return f"UniformLevelStorage({{{attrs}}}, device={self.device})" # --------------------------------------------------------------------------- # SegmentedLevelStorage (variable-length segments) # --------------------------------------------------------------------------- class SegmentedLevelStorage(BaseLevelStorage): """Container for segmented data with variable-length segments. Each "sample" corresponds to a segment of contiguous elements. The ``segment_lengths`` tensor records how many elements belong to each segment, and ``batch_idx`` / ``batch_ptr`` are lazily computed from it. Parameters ---------- data : dict[str, Tensor], optional Initial attribute tensors (all must have the same ``shape[0]``). device : DeviceType, optional Target device. attr_map : LevelSchema, optional Shared attribute registry. segment_lengths : list[int] or Tensor, optional Number of elements per segment. If ``None`` and *data* is provided, a single segment spanning the full data is assumed. batch_idx : Tensor, optional Pre-computed segment assignment per element. batch_ptr : Tensor, optional Pre-computed cumulative element counts (length = num_segments + 1). batch_ptr_capacity : int, optional If set, pre-allocate batch_ptr with this length so put() can append segments without reallocating. Ignored when batch_ptr is provided. validate : bool If ``True``, verify consistency between data and segment lengths. Raises ------ ValueError If segment lengths are negative or don't sum to ``data.shape[0]``. OverflowError If segment lengths or pointer values exceed the signed int32 range. """ def __init__( self, data: dict[str, Tensor] | None = None, device: DeviceType | None = None, attr_map: LevelSchema | None = None, segment_lengths: list[int] | Tensor | None = None, batch_idx: Tensor | None = None, batch_ptr: Tensor | None = None, batch_ptr_capacity: int | None = None, validate: bool = True, ) -> None: super().__init__(data, device, attr_map, validate) if data is not None and validate and not self._data.is_empty(): sizes = {self._data[k].shape[0] for k in self._data.keys()} if len(sizes) > 1: raise ValueError( f"All attributes must have the same first dimension, got {sizes}" ) if segment_lengths is None: if not self._data.is_empty(): num_el = self._data.shape[0] segment_lengths = [num_el] else: segment_lengths = [] self.segment_lengths, normalized_batch_ptr = _checked_segment_metadata( segment_lengths, self.device, batch_ptr=batch_ptr, batch_ptr_capacity=batch_ptr_capacity if batch_ptr is None else None, ) self._batch_idx: Tensor | None = ( batch_idx.to(device=self.device, dtype=torch.int32) if batch_idx is not None else None ) requested_batch_ptr_capacity: int | None = None if batch_ptr is not None: self._batch_ptr = normalized_batch_ptr requested_batch_ptr_capacity = self._batch_ptr.shape[0] elif batch_ptr_capacity is not None: self._batch_ptr = normalized_batch_ptr requested_batch_ptr_capacity = self._batch_ptr.shape[0] else: self._batch_ptr = None self._batch_ptr_capacity = requested_batch_ptr_capacity self._batch_ptr_np: np.ndarray | None = None self._segment_indices: Tensor | None = None if validate: self._validate_initialization() # -- Validation --------------------------------------------------------- def _validate_initialization(self) -> None: """Check segment lengths, batch_idx, and batch_ptr for consistency.""" total_elements = self.num_elements() if (self.segment_lengths < 0).any(): raise ValueError( f"Segment lengths cannot be negative: {self.segment_lengths}" ) if self._data.is_empty(): return segment_sum = self.segment_lengths.sum().item() if segment_sum != total_elements: raise ValueError( f"Sum of segment_lengths ({segment_sum}) != data length ({total_elements})" ) if self._batch_idx is not None: n_seg = len(self.segment_lengths) if self._batch_idx.shape[0] != total_elements: raise ValueError( f"batch_idx length {self._batch_idx.shape[0]} != data length {total_elements}" ) if total_elements > 0: if self._batch_idx[0] != 0: raise ValueError( f"batch_idx must start at 0, got {self._batch_idx[0].item()}" ) if self._batch_idx[-1] != n_seg - 1: raise ValueError( f"batch_idx last element {self._batch_idx[-1].item()} " f"!= num_segments - 1 ({n_seg - 1})" ) if self._batch_ptr is not None: expected_len = len(self.segment_lengths) + 1 if len(self._batch_ptr) < expected_len: raise ValueError( f"batch_ptr length {len(self._batch_ptr)} < num_segments + 1 ({expected_len})" ) if self._batch_ptr[0] != 0: raise ValueError( f"batch_ptr must start at 0, got {self._batch_ptr[0].item()}" ) logical_end = self._batch_ptr[expected_len - 1].item() if logical_end != total_elements: raise ValueError( f"batch_ptr logical end (index {expected_len - 1}) {logical_end} " f"!= data length ({total_elements})" ) # -- Special methods ---------------------------------------------------- def __len__(self) -> int: n = getattr(self, "_num_segments", None) if n is not None: return n return len(self.segment_lengths) def __getattr__(self, name: str) -> Any: """Lazy-initialise ``batch_idx`` and ``batch_ptr`` on first access.""" match name: case "batch_idx": if not hasattr(self, "_batch_idx") or self._batch_idx is None: self._lazy_init_batch_idx() return self._batch_idx case "batch_ptr": if not hasattr(self, "_batch_ptr") or self._batch_ptr is None: self._lazy_init_batch_ptr() return self._batch_ptr case _: raise AttributeError( f"'{type(self).__name__}' has no attribute '{name}'" ) def __repr__(self) -> str: attrs = ", ".join(f"'{k}': {self._data[k].shape}" for k in self._data.keys()) return ( f"SegmentedLevelStorage({{{attrs}}}, " f"segments={len(self.segment_lengths)}, device={self.device})" ) # -- Queries ------------------------------------------------------------ def num_elements(self) -> int: """Total number of elements (active count; after defrag, same as batch_ptr[-1]).""" if self._data.is_empty(): return 0 if len(self.segment_lengths) == 0: return 0 n = getattr(self, "_num_elements_kept", None) if n is not None: return n return self._data.shape[0] def is_segmented(self) -> bool: return True # -- Lazy initialisers -------------------------------------------------- def _lazy_init_batch_idx(self) -> None: if self._batch_idx is None: self._batch_idx = torch.repeat_interleave( torch.arange( len(self.segment_lengths), device=self.device, dtype=torch.int32 ), self.segment_lengths, ) @torch.compiler.disable def _lazy_init_batch_ptr(self) -> None: if self._batch_ptr is None: # segment_lengths were range-checked when set, so building the # pointer needs no second, host-synchronizing check. self.segment_lengths, self._batch_ptr = _checked_segment_metadata( self.segment_lengths, self.device, batch_ptr_capacity=self._batch_ptr_capacity, check=False, ) self._batch_ptr_capacity = ( self._batch_ptr.shape[0] if self._batch_ptr_capacity is not None else None ) def _replace_segment_lengths( self, segment_lengths: list[int] | Tensor, *, preserve_pointer: bool = True, normalized_metadata: tuple[Tensor, Tensor] | None = None, ) -> None: """Replace segment metadata, preserving pointer capacity when present.""" had_pointer_cache = preserve_pointer and self._batch_ptr is not None preserve_capacity = preserve_pointer and self._batch_ptr_capacity is not None pointer_capacity = self._batch_ptr_capacity if preserve_capacity else None if normalized_metadata is None: self.segment_lengths, pointer = _checked_segment_metadata( segment_lengths, self.device, batch_ptr_capacity=pointer_capacity, ) else: self.segment_lengths, pointer = normalized_metadata if had_pointer_cache or preserve_capacity: self._batch_ptr = pointer else: self._batch_ptr = None if preserve_capacity: self._batch_ptr_capacity = pointer.shape[0] self._batch_idx = None self._batch_ptr_np = None self._segment_indices = None if hasattr(self, "_num_segments"): object.__delattr__(self, "_num_segments") object.__delattr__(self, "_num_elements_kept") def _lazy_init_segment_indices(self) -> None: if self._segment_indices is None: self._segment_indices = torch.arange( len(self.segment_lengths), device=self.device, dtype=torch.int32 ) # -- Validation --------------------------------------------------------- def _validate_setitem(self, key: str, value: Tensor) -> None: if not self._data.is_empty() and len(value) != self.num_elements(): raise ValueError(f"Length mismatch: {len(value)} vs {self.num_elements()}") # -- Indexing ----------------------------------------------------------- def _normalize_segment_index(self, idx: IndexType) -> torch.Tensor: """Normalize *idx* to a 1-D tensor of segment indices.""" if isinstance(idx, int): i = idx + len(self) if idx < 0 else idx return torch.tensor([i], device=self.device, dtype=INDEX_DTYPE) if isinstance(idx, slice): start, stop, step = idx.indices(len(self)) return torch.arange( start, stop, step, device=self.device, dtype=INDEX_DTYPE ) if isinstance(idx, list): t = torch.as_tensor(idx, device=self.device) if t.dtype != INDEX_DTYPE and t.dtype != torch.bool: t = t.to(INDEX_DTYPE) elif isinstance(idx, torch.Tensor): t = idx.to(device=self.device) else: raise TypeError(f"Unsupported segment index type '{type(idx).__name__}'") if t.dtype == torch.bool: if t.numel() != len(self): raise ValueError( f"Boolean index length {t.numel()} != num segments {len(self)}" ) return torch.nonzero(t, as_tuple=False).flatten().to(torch.int64) return t def _expand_idx(self, idx: IndexType) -> Tensor: """Convert segment-level *idx* into element-level indices. Parameters ---------- idx : int, slice, or Tensor Segment-level index. Returns ------- Tensor Element-level indices on ``self.device``. """ match idx: case int(): self._lazy_init_batch_ptr() return torch.arange( self._batch_ptr[idx], self._batch_ptr[idx + 1], device=self.device, ) case slice(): start, stop, step = idx.indices(len(self.segment_lengths)) if step == 1: if start == 0 and stop == len(self.segment_lengths): return torch.arange( 0, self.num_elements(), device=self.device, dtype=torch.int64, ) else: self._lazy_init_batch_ptr() el_start = int(self._batch_ptr[start].item()) el_end = int(self._batch_ptr[stop].item()) return torch.arange( el_start, el_end, device=self.device, dtype=torch.int64 ) seg_idx = self._normalize_segment_index(idx) if self.device.type == "cuda": self._lazy_init_batch_ptr() return _expand_segments_warp(seg_idx, self._batch_ptr, torch.int64) else: starts = self.batch_ptr[seg_idx].to(torch.int64) ends = self.batch_ptr[seg_idx + 1].to(torch.int64) lengths = ends - starts total = int(torch.sum(lengths, dtype=torch.int64).item()) if total < 0 or total > _INT32_MAX: raise OverflowError( f"Selected element count {total} is outside the supported int32 range " f"({_INT32_MAX})" ) if total == 0: return torch.empty(0, device=self.device, dtype=torch.int64) repeated_starts = torch.repeat_interleave( starts, lengths, output_size=total ) cum_lengths = torch.cumsum(lengths, 0, dtype=torch.int64) prefix = torch.repeat_interleave( cum_lengths - lengths, lengths, output_size=total, ) local = torch.arange(total, device=self.device, dtype=torch.int64) - prefix return repeated_starts + local def _select_segment_lengths(self, idx: IndexType) -> Tensor: """Return segment_lengths for the selected segments.""" if isinstance(idx, int): return self.segment_lengths[idx : idx + 1] if isinstance(idx, slice): return self.segment_lengths[idx] return self.segment_lengths[to_tensor(idx).to(device=self.device)] def select( self, idx: IndexType, *, drop: Collection[str] = () ) -> SegmentedLevelStorage: """Select a subset of segments. Parameters ---------- idx : int, slice, or Tensor Segment-level index. drop : Collection[str], optional Keys left out of the selection instead of being copied. The segment lengths are selected regardless. Default ``()``. Returns ------- SegmentedLevelStorage """ element_idx = self._expand_idx(idx) segment_lengths = self._select_segment_lengths(idx) data = self._data.exclude(*drop) if drop else self._data selected_td = data[element_idx.to(device=self.device)] return self.__class__( selected_td, device=self.device, segment_lengths=segment_lengths, validate=False, ) def update_at(self, key: str, value: Any, idx: IndexType) -> None: """Update attribute *key* at the given segment indices. Parameters ---------- key : str Attribute name. value : Any Replacement values (converted to tensor). idx : IndexType Segment-level indices. Raises ------ KeyError If *key* is not present. """ if key not in self._data.keys(): raise KeyError(f"Key '{key}' not found") element_idx = self._expand_idx(idx) self._data[key][element_idx.to(device=self.device)] = to_tensor( value, self.device ) # -- Device / copy ------------------------------------------------------ def to_device( self, device: DeviceType, *, non_blocking: bool = False ) -> SegmentedLevelStorage: """Move all tensors (including bookkeeping) to *device*. Returns ------- Self For method chaining. """ super().to_device(device, non_blocking=non_blocking) self.segment_lengths = self.segment_lengths.to( device, non_blocking=non_blocking ) if self._batch_idx is not None: self._batch_idx = self._batch_idx.to(device, non_blocking=non_blocking) if self._batch_ptr is not None: self._batch_ptr = self._batch_ptr.to(device, non_blocking=non_blocking) if self._segment_indices is not None: self._segment_indices = self._segment_indices.to( device, non_blocking=non_blocking ) self._batch_ptr_np = None return self def clone(self, *, drop: Collection[str] = ()) -> SegmentedLevelStorage: """Return a deep copy with independent tensors, segment info, and attr_map. Parameters ---------- drop : Collection[str], optional Keys left out of the copy instead of being cloned. The segment lengths, batch index, and batch pointer are cloned regardless. Default ``()``. """ cloned_data = { k: _clone_tensor(self._data[k]) for k in self._data.keys() if k not in drop } cloned_seg = _clone_tensor(self.segment_lengths) cloned_bidx = ( _clone_tensor(self._batch_idx) if self._batch_idx is not None else None ) cloned_bptr = ( _clone_tensor(self._batch_ptr) if self._batch_ptr is not None else None ) out = self.__class__( data=cloned_data, device=self.device, attr_map=self._attr_map.clone(), segment_lengths=cloned_seg, batch_idx=cloned_bidx, batch_ptr=cloned_bptr, validate=False, ) if cloned_bptr is None: object.__setattr__(out, "_batch_ptr_capacity", self._batch_ptr_capacity) if getattr(self, "_num_segments", None) is not None: object.__setattr__(out, "_num_segments", self._num_segments) object.__setattr__(out, "_num_elements_kept", self._num_elements_kept) return out # -- Concatenation ------------------------------------------------------ def concatenate( self, other: SegmentedLevelStorage, strict: bool = False, ) -> SegmentedLevelStorage: """Concatenate *other* into this container (in-place). Parameters ---------- other : SegmentedLevelStorage Container to append. strict : bool If ``True``, require identical attribute sets. Returns ------- Self For method chaining. Raises ------ ValueError On key mismatch (strict) or incompatible trailing shapes. OverflowError If the combined segmented pointer exceeds the signed int32 range. """ self_keys = set(self.keys()) other_keys = set(other.keys()) common = self_keys & other_keys if strict and self_keys != other_keys: raise ValueError(f"Keys mismatch: {self_keys} vs {other_keys}") if not common: return self combined_lengths = torch.cat( [self.segment_lengths, other.segment_lengths.to(self.device)] ) _checked_segment_metadata( combined_lengths, self.device, batch_ptr_capacity=self._batch_ptr_capacity, ) # TensorDict enforces batch_size; replace with new TensorDict of concatenated data new_data = {} for key in common: other_tensor = to_tensor(other[key], self.device) if not _validate_trailing_shapes([self[key], other_tensor]): raise ValueError( f"Key '{key}' has incompatible shape: " f"{self[key].shape} vs {other_tensor.shape}" ) new_data[key] = _concatenate_tensors(self._data[key], other_tensor) new_total = next(iter(new_data.values())).shape[0] self._data = TensorDict( new_data, batch_size=[new_total], device=self.device, ) self._replace_segment_lengths(combined_lengths) return self def extend_for_appended_graphs(self, n: int) -> SegmentedLevelStorage: """Extend segment count by *n* zero-length segments (for batches that lack this group). Used when appending a batch that does not have this group: segment_lengths is extended with *n* zeros so graph counts stay aligned. No new elements are added to the data tensors. Parameters ---------- n : int Number of graph slots (segments of length 0) to add. Returns ------- Self For method chaining. """ if n <= 0: return self extra = torch.zeros(n, device=self.device, dtype=self.segment_lengths.dtype) self._replace_segment_lengths(torch.cat([self.segment_lengths, extra])) return self def compute_put_per_system_fit_mask( self, source: SegmentedLevelStorage, source_mask: Tensor, fit_mask: Tensor, ) -> None: """Compute which source segments would fit in this storage; write result into fit_mask in place. Parameters ---------- source : SegmentedLevelStorage Source storage; len(source) gives num_systems (segments). source_mask : Tensor, shape (num_systems,), dtype bool True for each segment considered for copy. fit_mask : Tensor, shape (num_systems,), dtype bool Output written in place: True where that segment fits in this storage. Notes ----- No data is copied. Each copied segment needs a free slot in this storage's batch_ptr, so only the first selected segments up to the free slot count can fit. Use with put after combining (e.g. logical_and) with other levels' fit masks. Fieldless source and destination groups check pointer capacity and whether cumulative cardinality remains within the signed int32 range. """ n_seg = len(source) if source_mask.shape[0] != n_seg: raise ValueError( f"source_mask shape {source_mask.shape[0]} != len(source) {n_seg}" ) if fit_mask.shape[0] != n_seg: raise ValueError(f"fit_mask shape {fit_mask.shape[0]} != {n_seg}") source_mask = source_mask.to(device=self.device, dtype=torch.bool) fit_mask = fit_mask.to(device=self.device, dtype=torch.bool) source_fieldless = source._data.is_empty() dest_fieldless = self._data.is_empty() num_dest_segments = len(self) if source_fieldless != dest_fieldless: raise ValueError( "Segmented put requires both source and destination to be " "fieldless or both to contain payload data" ) source._lazy_init_batch_ptr() self._lazy_init_batch_ptr() # The -2 reserves the current end boundary and the duplicate final end # written by _copy_masked_segmented_kernel. free_slots = self._batch_ptr.shape[0] - num_dest_segments - 2 if free_slots <= 0: fit_mask.zero_() return source_lengths = ( source._batch_ptr[1 : n_seg + 1] - source._batch_ptr[:n_seg] ).to(torch.int64) masked_lengths = torch.where( source_mask, source_lengths, torch.zeros_like(source_lengths), ) ends = self._batch_ptr[num_dest_segments].to(torch.int64) + torch.cumsum( masked_lengths, dim=0, dtype=torch.int64 ) dest_capacity = ( _INT32_MAX if source_fieldless else min(self._data.shape[0], _INT32_MAX) ) within_slots = torch.cumsum(source_mask, dim=0) <= free_slots fit_mask.copy_( source_mask & within_slots & (ends >= 0) & (ends <= dest_capacity) ) def put( self, src: SegmentedLevelStorage, mask: Tensor, *, copied_mask: Tensor | None = None, num_selected: int | None = None, ) -> None: """Put segments where mask[i] is True from src into this storage (buffer). New segment boundaries are appended to this storage's batch_ptr. Supported payload dtypes are bool, float32, float64, int32, and int64; source and destination dtypes must match. Fieldless source and destination groups copy segment metadata only. Only the first selected segments that fit the data capacity and the free batch_ptr slots are copied. If *copied_mask* is provided, it is updated in place for each copied segment. Parameters ---------- src : SegmentedLevelStorage Source storage; same attribute keys as self. mask : Tensor (num_segments,) bool, True = copy this segment. copied_mask : Tensor, optional (num_segments,) bool; if provided, modified in place with which segments were actually copied. If None, stored on *src* as ``_copied_mask`` for use by :meth:`defrag`. num_selected : int, optional Number of True entries in *mask* when every selected segment is known to fit, as after :meth:`compute_put_per_system_fit_mask`. Skips the int32 range check and reading the copied count back from the device. Raises ------ OverflowError If *num_selected* is not given and the resulting segmented pointer would exceed the signed int32 range. """ source_fieldless = src._data.is_empty() dest_fieldless = self._data.is_empty() if source_fieldless != dest_fieldless: raise ValueError( "Segmented put requires both source and destination to be " "fieldless or both to contain payload data" ) if source_fieldless: n_seg = len(src) if mask.shape[0] != n_seg: raise ValueError(f"mask shape {mask.shape[0]} != num segments {n_seg}") mask = mask.to(device=self.device, dtype=torch.bool) num_dest_segments = len(self) src._lazy_init_batch_ptr() self._lazy_init_batch_ptr() free_slots = self._batch_ptr.shape[0] - num_dest_segments - 2 source_lengths = src.segment_lengths[:n_seg].to(self.device) if num_selected is None: # Update mask to allow partial copies up to the free pointer slots. mask = mask & (torch.cumsum(mask, dim=0) <= free_slots) selected_lengths = source_lengths[mask] else: # Gather the selected lengths into a known-size tensor; boolean # indexing would read the count back from the device. rank = torch.cumsum(mask, dim=0) - 1 selected_lengths_buffer = source_lengths.new_zeros(num_selected + 1) selected_lengths_buffer.scatter_( 0, torch.where(mask, rank, num_selected), source_lengths ) selected_lengths = selected_lengths_buffer[:num_selected] old_lengths = self.segment_lengths[:num_dest_segments] combined_lengths = torch.cat([old_lengths, selected_lengths]) normalized_metadata = _checked_segment_metadata( combined_lengths, self.device, batch_ptr_capacity=self._batch_ptr_capacity, check=num_selected is None, ) if copied_mask is not None: if copied_mask.shape[0] != n_seg: raise ValueError( f"copied_mask shape {copied_mask.shape[0]} != {n_seg}" ) out_mask = copied_mask.to(device=self.device, dtype=torch.bool) else: out_mask = torch.zeros(n_seg, device=self.device, dtype=torch.bool) object.__setattr__(src, "_copied_mask", out_mask) if free_slots <= 0: out_mask.zero_() return self._replace_segment_lengths( combined_lengths, preserve_pointer=True, normalized_metadata=normalized_metadata, ) out_mask.copy_(mask) return if self._data.is_empty() or src._data.is_empty(): raise ValueError("put requires non-empty source and dest") fields = _validate_buffer_field_pairs(self._data, src._data) if not fields: raise ValueError("put requires at least one common attribute") n_seg = len(src) if mask.shape[0] != n_seg: raise ValueError(f"mask shape {mask.shape[0]} != num segments {n_seg}") mask = mask.to(device=self.device, dtype=torch.bool) num_dest_segments = len(self) if num_selected is None: source_lengths = src.segment_lengths[:n_seg].to(self.device) selected_lengths = source_lengths[mask] old_lengths = self.segment_lengths[:num_dest_segments] _checked_segment_metadata( torch.cat([old_lengths, selected_lengths]), self.device ) if copied_mask is not None: if copied_mask.shape[0] != n_seg: raise ValueError(f"copied_mask shape {copied_mask.shape[0]} != {n_seg}") out_mask = copied_mask.to(device=self.device, dtype=torch.bool) else: out_mask = torch.zeros(n_seg, device=self.device, dtype=torch.bool) object.__setattr__(src, "_copied_mask", out_mask) src._lazy_init_batch_ptr() self._lazy_init_batch_ptr() dest_batch_ptr = self._batch_ptr # Require room for at least one new segment. if dest_batch_ptr.shape[0] <= num_dest_segments + 2: return # A buffer source keeps spare pointer slots past its n_seg segments. # The kernels size their launch from the pointer they are given, so # pass only the occupied boundaries to match the n_seg-long mask. src_batch_ptr = src._batch_ptr[: n_seg + 1] first_key, *remaining_keys = fields new_num_dest = put_masked_segmented( src._data[first_key], src_batch_ptr, mask, self._data[first_key], dest_batch_ptr, num_dest_segments, out_mask, ) for key in remaining_keys: src_t = src._data[key] dest_t = self._data[key] put_masked_segmented( src_t, src_batch_ptr, mask, dest_t, dest_batch_ptr, num_dest_segments, out_mask, ) if new_num_dest is not None: if num_selected is None: new_n = int(new_num_dest.item()) else: new_n = num_dest_segments + num_selected # Pad unused slots with the final boundary and reuse the pointer to avoid # rebuilding it and another host sync. dest_batch_ptr[new_n + 1 :] = dest_batch_ptr[new_n] lengths = dest_batch_ptr[1 : new_n + 1] - dest_batch_ptr[:new_n] self._replace_segment_lengths( lengths, preserve_pointer=True, normalized_metadata=(lengths, dest_batch_ptr), ) def defrag( self, copied_mask: Tensor | None = None, ) -> SegmentedLevelStorage: """Defrag in-place: remove segments where copied_mask[i] is True. Kept segments move to the front and pointer metadata is updated in place without trimming allocated capacity. Supported payload dtypes are bool, float32, float64, int32, and int64. All fields are validated before compaction. Fieldless groups compact only segment metadata. Parameters ---------- copied_mask : Tensor, optional (num_segments,) bool; if None, uses ``_copied_mask`` from the last :meth:`put`. Returns ------- Self For method chaining. """ if self._data.is_empty(): n_seg = len(self) if copied_mask is None: copied_mask = getattr(self, "_copied_mask", None) if copied_mask is None: raise ValueError("defrag requires copied_mask or a prior put") else: copied_mask = copied_mask.to(device=self.device, dtype=torch.bool) if copied_mask.shape[0] != n_seg: raise ValueError(f"copied_mask shape {copied_mask.shape[0]} != {n_seg}") kept_lengths = self.segment_lengths[:n_seg][~copied_mask] self._replace_segment_lengths(kept_lengths, preserve_pointer=True) self._lazy_init_batch_ptr() object.__setattr__(self, "_num_segments", len(kept_lengths)) object.__setattr__( self, "_num_elements_kept", int(self._batch_ptr[len(kept_lengths)].item()), ) if hasattr(self, "_copied_mask"): object.__delattr__(self, "_copied_mask") return self n_seg = len(self) if copied_mask is None: copied_mask = getattr(self, "_copied_mask", None) if copied_mask is None: raise ValueError("defrag requires copied_mask or a prior put") else: copied_mask = copied_mask.to(device=self.device, dtype=torch.bool) if copied_mask.shape[0] != n_seg: raise ValueError(f"copied_mask shape {copied_mask.shape[0]} != {n_seg}") self._lazy_init_batch_ptr() keys = _validate_buffer_field_dtypes(self._data) original_bp = self._batch_ptr.clone() num_kept_t = defrag_segmented( self._data[keys[0]], self._batch_ptr, copied_mask.clone() ) for key in keys[1:]: defrag_segmented(self._data[key], original_bp.clone(), copied_mask.clone()) n_kept = int(num_kept_t.item()) self._replace_segment_lengths( self._batch_ptr[1 : n_kept + 1] - self._batch_ptr[:n_kept], preserve_pointer=True, ) object.__setattr__(self, "_num_segments", n_kept) object.__setattr__( self, "_num_elements_kept", int(self._batch_ptr[n_kept].item()) ) # Kernel already compacted each tensor in place (kept rows at front, rest zeroed); # buffer shape is unchanged for fixed-size batches. if hasattr(self, "_copied_mask"): object.__delattr__(self, "_copied_mask") return self # --------------------------------------------------------------------------- # MultiLevelStorage # --------------------------------------------------------------------------- class MultiLevelStorage: """Multi-group container that routes attribute access to the correct group. Manages a dict of :class:`UniformLevelStorage` and :class:`SegmentedLevelStorage` instances, exposing a flat attribute namespace across all groups. Parameters ---------- groups : dict[str, UniformLevelStorage | SegmentedLevelStorage], optional Named groups. ``None`` creates an empty batch. validate : bool If ``True``, check length/device consistency and duplicate attributes. attr_map : LevelSchema, optional Shared attribute registry. device : DeviceType, optional Target device. If ``None``, inferred from existing groups. Raises ------ ValueError On inconsistent group lengths or duplicate attribute names. """ def __init__( self, groups: dict[str, UniformLevelStorage | SegmentedLevelStorage] | None = None, validate: bool = True, attr_map: LevelSchema | None = None, device: DeviceType | None = None, ) -> None: self.groups: dict[str, UniformLevelStorage | SegmentedLevelStorage] = ( groups if groups is not None else {} ) self.attr_map = attr_map if attr_map is not None else LevelSchema() self.device = resolve_device(device) if validate and self.groups: self._validate_consistency() if device is not None: self.to_device(device) elif self.groups: self._infer_and_set_device() # -- Validation --------------------------------------------------------- def _infer_and_set_device(self) -> None: first_device = next(iter(self.groups.values())).device self.to_device(first_device) def _validate_consistency(self) -> None: self._validate_length_consistency() self._validate_no_duplicate_attributes() self._validate_consistent_devices() def _validate_length_consistency(self) -> None: if not self.groups: return expected = len(next(iter(self.groups.values()))) for name, group in self.groups.items(): if len(group) != expected: raise ValueError( f"Group '{name}' has length {len(group)}, expected {expected}" ) def _validate_no_duplicate_attributes(self) -> None: seen: set[str] = set() for group_name, group in self.groups.items(): for key in group.keys(): if key in seen: raise ValueError( f"Attribute '{key}' is duplicated across storage groups; " f"found again in group '{group_name}'" ) seen.add(key) def _validate_consistent_devices(self) -> None: if not self.groups: return first_device = next(iter(self.groups.values())).device for group in self.groups.values(): group.to_device(first_device) # -- Length / properties ------------------------------------------------ def __len__(self) -> int: if not self.groups: return 0 return len(next(iter(self.groups.values()))) @property def num_systems(self) -> int: """Number of systems from the ``'system'`` group. Raises ------ ValueError If no ``'system'`` group exists. """ if "system" not in self.groups: raise ValueError("'system' group not found in batch") return len(self.groups["system"]) @property def num_atoms(self) -> int: """Total atom count from the ``'atoms'`` group.""" group = self.groups.get("atoms") if group is None: return 0 if isinstance(group, SegmentedLevelStorage): return group.num_elements() first_val = next(iter(group._data.values())) return first_val.shape[0] * first_val.shape[1] # -- Group lookup ------------------------------------------------------- def _group_name_from_attr(self, attr_name: str) -> str | None: for group_name, group in self.groups.items(): if attr_name in group: return group_name return None def group_from_attr( self, attr_name: str ) -> UniformLevelStorage | SegmentedLevelStorage | None: """Return the group that contains *attr_name*, or ``None``.""" name = self._group_name_from_attr(attr_name) return self.groups[name] if name is not None else None # -- Dict-like interface ------------------------------------------------ def __getitem__(self, key: str | IndexType) -> Tensor | MultiLevelStorage: """Access an attribute by name or select a subset by index.""" if isinstance(key, str): group_name = self._group_name_from_attr(key) if group_name is None: raise KeyError(f"Attribute '{key}' not found in batch") return self.groups[group_name][key] return self.select(key) def __setitem__(self, key: str, value: Any) -> None: """Set an attribute, routing to the group that holds it or ``attr_map``.""" group_name = self._group_name_from_attr(key) if group_name is None: try: group_name = self.attr_map.group(key) except KeyError: group_name = "system" # Unknown keys default to system-level dtype = self.attr_map.dtypes.get(key, None) tensor = to_tensor(value, device=self.device, dtype=dtype) if group_name in self.groups: self.groups[group_name][key] = tensor else: if self.attr_map.is_segmented_group(group_name): raise ValueError( f"Group '{group_name}' is segmented but not found in batch" ) self.groups[group_name] = UniformLevelStorage( data={key: tensor}, device=self.device, attr_map=self.attr_map, validate=False, ) def __delitem__(self, key: str) -> None: group_name = self._group_name_from_attr(key) if group_name is None: raise KeyError(f"Attribute '{key}' not found in batch") del self.groups[group_name][key] def __contains__(self, key: str) -> bool: return any(key in g for g in self.groups.values()) def __iter__(self) -> Iterator[str]: for group in self.groups.values(): yield from group def __repr__(self) -> str: return f"MultiLevelStorage(groups={self.groups}, attr_map={self.attr_map})" def __str__(self) -> str: return repr(self) def keys(self) -> list[str]: """All attribute names across groups.""" return [k for g in self.groups.values() for k in g.keys()] def values(self) -> list[Tensor]: """All attribute tensors across groups.""" return [v for g in self.groups.values() for v in g.values()] def items(self) -> Iterator[tuple[str, Tensor]]: """Iterate over ``(attr_name, tensor)`` across all groups.""" for group in self.groups.values(): yield from group.items() def get(self, key: str, default: Tensor | None = None) -> Tensor | None: """Return the tensor for *key*, or *default* if absent.""" group_name = self._group_name_from_attr(key) if group_name is None: return default return self.groups[group_name].get(key, default) def pop(self, key: str, default: Any = None) -> Tensor: """Remove and return the tensor for *key*. Raises ------ KeyError If *key* is absent and no *default* was given. """ group_name = self._group_name_from_attr(key) if group_name is None: if default is not None: return default raise KeyError(f"Attribute '{key}' not found in batch") return self.groups[group_name].pop(key, default) # -- Selection / cloning ------------------------------------------------ def select( self, idx: IndexType, *, drop: Collection[str] = () ) -> MultiLevelStorage: """Select a subset of samples across all groups. Parameters ---------- idx : IndexType Sample-level index specification. drop : Collection[str], optional Keys left out of every group's selection instead of being copied. Default ``()``. Returns ------- MultiLevelStorage """ if isinstance(idx, int): idx = slice(idx, idx + 1) return self.__class__( groups={k: v.select(idx, drop=drop) for k, v in self.groups.items()}, attr_map=self.attr_map, validate=False, ) def is_segmented(self) -> bool: """Return ``True`` if any group is segmented.""" return any(g.is_segmented() for g in self.groups.values()) def to_device( self, device: DeviceType, *, non_blocking: bool = False ) -> MultiLevelStorage: """Move all groups to *device*. Parameters ---------- device : DeviceType Target device. A bare ``"cuda"`` is recorded as the CUDA device current at the time of the move, which is where the tensors land. non_blocking : bool, default False Whether tensor copies may be asynchronous when supported. Returns ------- Self For method chaining. """ device = resolve_device(device) self.device = device for group in self.groups.values(): group.to_device(device, non_blocking=non_blocking) return self def clone(self, *, drop: Collection[str] = ()) -> MultiLevelStorage: """Return a deep copy with independent groups and attr_map. Parameters ---------- drop : Collection[str], optional Keys left out of every group's copy instead of being cloned. Default ``()``. """ return self.__class__( groups={n: g.clone(drop=drop) for n, g in self.groups.items()}, validate=False, attr_map=self.attr_map.clone(), device=self.device, ) def update_at(self, key: str, value: Any, idx: IndexType) -> None: """Update attribute *key* at the given indices. Parameters ---------- key : str Attribute name. value : Any Replacement values. idx : IndexType Target indices. Raises ------ KeyError If *key* is not found in any group. """ group_name = self._group_name_from_attr(key) if group_name is None: raise KeyError(f"Attribute '{key}' not found in batch") dtype = self.attr_map.dtypes.get(key, None) tensor = to_tensor(value, device=self.device, dtype=dtype) self.groups[group_name].update_at(key, tensor, idx) def concatenate( self, other: MultiLevelStorage, strict: bool = False ) -> MultiLevelStorage: """Concatenate with another batch. Parameters ---------- other : MultiLevelStorage Data to append. strict : bool If ``True``, require identical attribute sets. Returns ------- MultiLevelStorage """ return self.from_batches([self, other], strict=strict) # -- Factory methods ---------------------------------------------------- @classmethod def from_data( cls, data: dict[str, Tensor], attr_map: LevelSchema | None = None, segment_lengths: dict[str, list[int] | Tensor] | None = None, device: DeviceType | None = "cpu", validate: bool = True, ) -> MultiLevelStorage: """Create an :class:`MultiLevelStorage` from a flat attribute dict. Parameters ---------- data : dict[str, Tensor] Attribute tensors keyed by name. attr_map : LevelSchema, optional Registry used to route attributes to groups. segment_lengths : dict[str, list[int] | Tensor], optional Per-group segment lengths for segmented groups. device : DeviceType, optional Target device. validate : bool Whether to validate consistency. Returns ------- MultiLevelStorage Raises ------ ValueError If a segmented group has no corresponding entry in *segment_lengths*. """ if attr_map is None: attr_map = LevelSchema() grouped: dict[str, dict[str, Tensor]] = defaultdict(dict) for key, value in data.items(): grouped[attr_map.group(key)][key] = value groups: dict[str, UniformLevelStorage | SegmentedLevelStorage] = {} for group_name, group_dict in grouped.items(): if segment_lengths is not None and attr_map.is_segmented_group(group_name): if group_name not in segment_lengths: raise ValueError( f"Segment lengths not provided for segmented group '{group_name}'" ) groups[group_name] = SegmentedLevelStorage( data=group_dict, device=device, segment_lengths=segment_lengths[group_name], validate=validate, attr_map=attr_map, ) else: groups[group_name] = UniformLevelStorage( data=group_dict, device=device, validate=validate, attr_map=attr_map ) return cls(groups=groups, attr_map=attr_map) def to_segmented(self, validate: bool = True) -> MultiLevelStorage: """Convert uniform groups into segmented format. Attributes belonging to segmented groups are flattened along dims 0-1, and uniform segment lengths are inferred from the original shape. Parameters ---------- validate : bool Whether to validate the resulting batch. Returns ------- MultiLevelStorage """ flat_data: dict[str, Tensor] = {} seg_lengths: dict[str, Tensor] = {} group_shapes: dict[str, tuple[int, int]] = {} device: DeviceType = "cpu" for attr_name, tensor in self.items(): if self.attr_map.is_segmented_attr(attr_name): group_name = self.attr_map.attr_to_group[attr_name] batch_size, num_atoms = tensor.shape[:2] if group_name in group_shapes: if group_shapes[group_name][0] != batch_size: raise ValueError( f"Batch size mismatch for group '{group_name}': " f"{group_shapes[group_name]} vs ({batch_size}, {num_atoms}) " f"for key '{attr_name}'" ) else: group_shapes[group_name] = (batch_size, num_atoms) tensor = tensor.flatten(0, 1) if group_name not in seg_lengths: seg_lengths[group_name] = torch.full( (batch_size,), num_atoms, device=tensor.device, dtype=torch.int32, ) flat_data[attr_name] = tensor device = tensor.device return self.__class__.from_data( data=flat_data, attr_map=self.attr_map, segment_lengths=seg_lengths, device=device, validate=validate, ) @classmethod def from_batches( cls, batches: list[MultiLevelStorage], strict: bool = False, allow_conversion: bool = True, ) -> MultiLevelStorage: """Merge multiple batches into one. Parameters ---------- batches : list[MultiLevelStorage] Instances to combine. strict : bool If ``True``, require identical attribute sets. allow_conversion : bool If ``True``, automatically convert uniform batches to segmented format when mixing with segmented batches. Returns ------- MultiLevelStorage Notes ----- Uses the ``attr_map`` and the device from the first batch. Every contributed tensor, segment lengths included, is moved to that device before it is concatenated, so batches held on different devices merge. """ if not batches: return cls(attr_map=LevelSchema()) if len(batches) == 1: return batches[0] attr_map = batches[0].attr_map common_attrs = set.intersection(*(set(b.keys()) for b in batches)) if strict and common_attrs != set(batches[0].keys()): raise ValueError( f"Attribute sets differ. Batch 0: {set(batches[0].keys())}, " f"common: {common_attrs}" ) if not common_attrs: return cls(attr_map=attr_map) device = batches[0].device need_conversion = False if allow_conversion: seg_flags = [b.is_segmented() for b in batches] need_conversion = any(seg_flags) if not need_conversion: for attr in common_attrs: vals = [b[attr] for b in batches] if not cls._compatible_shapes_uniform(vals, seg_flags): need_conversion = True break if need_conversion: batches = [ b.to_segmented(validate=True) if not b.is_segmented() else b for b in batches ] merged_data: dict[str, Tensor] = {} merged_seg_lengths: dict[str, Tensor] = {} for attr in common_attrs: dtype_str = attr_map.dtypes.get(attr, None) tensors = [ to_tensor(b[attr], dtype=dtype_str, device=device) for b in batches ] merged_data[attr] = torch.cat(tensors, dim=0) attr_groups = [b.group_from_attr(attr) for b in batches] if all(g.is_segmented() for g in attr_groups): group_name = attr_map.group(attr) merged_seg_lengths[group_name] = torch.cat( [g.segment_lengths.to(device) for g in attr_groups], dim=0 ) return cls.from_data( data=merged_data, attr_map=attr_map, segment_lengths=merged_seg_lengths or None, device=device, ) @staticmethod def _compatible_shapes_uniform( tensors: list[Tensor], is_segmented: list[bool], ) -> bool: """Check whether all tensors can be concatenated without segmentation. Returns ``False`` if any tensor comes from a segmented source or if trailing shapes differ. """ if not tensors: return True trailing = None for tensor, seg in zip(tensors, is_segmented): if seg: return False if trailing is None: trailing = tensor.shape[1:] if tensor.shape[1:] != trailing: return False return True