# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""LoRA wrappers for interatomic potential models.
This module extends the underlying LoRA registry with wrappers for e3nn layers
used by E(3) equivariant interatomic potentials.
"""
from __future__ import annotations
import warnings
from collections.abc import Iterator
from contextlib import contextmanager
from functools import partial
from typing import Any, Literal, TypeAlias
import torch
from torch import nn
from torch.func import functional_call
from nvalchemi.training.peft import _peft
LoRALayer = _peft.LoRALayer
LoRALinear = _peft.LoRALinear
_TRANSFORMER_ENGINE_LORA_LINEAR = getattr(
_peft._physicsnemo_peft.lora, "LoRA_te_Linear", None
)
if _TRANSFORMER_ENGINE_LORA_LINEAR is not None:
TransformerEngineLoRALinear = _TRANSFORMER_ENGINE_LORA_LINEAR
LoRAWrappableLayer: TypeAlias = type[nn.Module]
LoRAWrapper: TypeAlias = type[_peft.LoRALayer]
LoRAWrapperRegistrations: TypeAlias = tuple[tuple[LoRAWrappableLayer, LoRAWrapper], ...]
__all__ = [
"CuEquivarianceLoRALinear",
"E3NNFullyConnectedLoRALayer",
"EquivariantLoRALinear",
"LoRALayer",
"LoRALinear",
"LoRAWrapper",
"LoRAWrapperRegistrations",
"LoRAWrappableLayer",
"available_lora_wrappers",
]
if _TRANSFORMER_ENGINE_LORA_LINEAR is not None:
__all__.append("TransformerEngineLoRALinear")
def __getattr__(name: str) -> object:
"""Lazily expose optional wrapper classes."""
if name == "TransformerEngineLoRALinear":
raise ImportError(
"TransformerEngineLoRALinear requires transformer_engine to be installed."
)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def _import_e3nn() -> tuple[type[nn.Module], type[nn.Module], object, type[nn.Module]]:
"""Import e3nn objects needed by the equivariant LoRA wrappers."""
try:
if hasattr(torch.serialization, "add_safe_globals"):
torch.serialization.add_safe_globals([slice])
from e3nn import o3
from e3nn.nn import Dropout
from e3nn.nn._fc import _Layer
except ImportError as exc:
raise ImportError(
"Equivariant LoRA requires e3nn. Install the e3nn dependency."
) from exc
return o3.Linear, _Layer, o3, Dropout
def _e3nn_layer_cls(kind: Literal["linear", "fully_connected"]) -> type[nn.Module]:
"""Return an e3nn layer class by kind."""
linear_cls, fc_layer_cls, _o3, _dropout_cls = _import_e3nn()
if kind == "linear":
return linear_cls
elif kind == "fully_connected":
return fc_layer_cls
else:
raise ValueError(f"Invalid e3nn layer kind: {kind}")
def _cueq_linear_cls() -> type[nn.Module]:
"""Return the cuEquivariance linear layer class."""
try:
from cuequivariance_torch.operations.linear import Linear
except ImportError as exc:
raise ImportError(
"cuEquivariance LoRA requires cuequivariance-torch. Install the "
"CUDA-aligned cuEq dependencies."
) from exc
return Linear
class EquivariantLoRALinear(nn.Module, LoRALayer):
"""LoRA layer for equivariant linear layers ``e3nn.o3.Linear``.
The adapter is itself equivariant: ``base(x) + scale * B(A(x))`` where
``A`` and ``B`` are ``e3nn.o3.Linear`` maps through irreps common to the
base layer's input and output.
Parameters
----------
base_layer : nn.Module
An ``e3nn.o3.Linear`` instance to wrap.
rank : int
Adapter multiplicity per irrep.
alpha : float
LoRA scaling parameter.
dropout : float, optional
Adapter dropout probability. Defaults to ``0.0``.
init : _peft.LoRAInit, optional
Initialization for the input-side adapter weights (``lora_A``).
``"default"`` uses a small Gaussian (``normal_(std=1e-3)``);
pass a callable ``(tensor) -> None`` for a custom in-place scheme.
``lora_B`` always starts at zero.
**kwargs : Any
Additional keyword arguments.
Attributes
----------
mergeable : bool
Required by ``LoRALayer``; whether this adapter can be folded into the
wrapped base layer.
enabled : bool
Required by ``LoRALayer``; whether the LoRA residual is active during
forward passes.
Properties
----------
lora_A : nn.Parameter
Required by ``LoRALayer``; weight parameter of the input-side equivariant adapter layer.
lora_B : nn.Parameter
Required by ``LoRALayer``; weight parameter of the output-side equivariant adapter layer.
"""
mergeable = True
def __init__(
self,
base_layer: nn.Module,
rank: int,
alpha: float,
dropout: float = 0.0,
init: _peft.LoRAInit = "default",
**kwargs: Any,
) -> None:
nn.Module.__init__(self)
linear_cls, _fc_layer_cls, _o3, dropout_cls = _import_e3nn()
if not isinstance(base_layer, linear_cls):
raise TypeError(
"EquivariantLoRALinear can only wrap e3nn.o3.Linear layers."
)
self.base_layer = base_layer
for param in self.base_layer.parameters():
param.requires_grad = False
self.enabled = True
# Initialize LoRA adapter layers.
self.rank = int(rank)
self.alpha = float(alpha)
self.scaling = self.alpha / self.rank
self.irreps_in = base_layer.irreps_in
self.irreps_out = base_layer.irreps_out
self.adapter_irreps = self._build_adapter_irreps(
self.irreps_in, self.irreps_out, self.rank
)
self.lora_A_layer = linear_cls(
self.irreps_in,
self.adapter_irreps,
internal_weights=True,
shared_weights=True,
biases=False,
)
self.lora_B_layer = linear_cls(
self.adapter_irreps,
self.irreps_out,
internal_weights=True,
shared_weights=True,
biases=False,
)
self.lora_A_layer.to(
device=base_layer.weight.device, dtype=base_layer.weight.dtype
)
self.lora_B_layer.to(
device=base_layer.weight.device, dtype=base_layer.weight.dtype
)
self.lora_dropout = (
dropout_cls(self.irreps_in, p=dropout) if dropout > 0.0 else nn.Identity()
)
# Initialize LoRA weights via ``init`` while keeping the adapter
# residual at zero (``lora_B`` starts at zero).
with torch.no_grad():
if callable(init):
init(self.lora_A_layer.weight)
elif init == "default":
self.lora_A_layer.weight.normal_(mean=0.0, std=1e-3)
else:
raise ValueError(
f"unknown init strategy {init!r}; use 'default' or a callable."
)
self.lora_B_layer.weight.zero_()
# Pre-compute instruction maps for efficient merged weight computation.
self._build_instruction_maps()
@property
def lora_A(self) -> nn.Parameter:
"""Return the input-side LoRA parameter."""
return self.lora_A_layer.weight
@property
def lora_B(self) -> nn.Parameter:
"""Return the output-side LoRA parameter."""
return self.lora_B_layer.weight
@classmethod
def is_compatible(cls, base_layer: nn.Module) -> bool:
"""Return whether ``base_layer`` has shared input/output irreps."""
try:
cls._build_adapter_irreps(
base_layer.irreps_in,
base_layer.irreps_out,
rank=1,
)
except (AttributeError, ValueError):
return False
return True
@staticmethod
def _build_adapter_irreps(
irreps_in: object, irreps_out: object, rank: int
) -> object:
"""Build equivariant adapter irreps from input/output overlap.
Following MACE's convention, each shared irrep type receives ``rank``
copies.
"""
_linear_cls, _fc_layer_cls, o3, _dropout_cls = _import_e3nn()
irreps_in = o3.Irreps(irreps_in).simplify()
irreps_out = o3.Irreps(irreps_out).simplify()
in_irrep_types = {ir for _, ir in irreps_in}
out_irrep_types = {ir for _, ir in irreps_out}
if len(in_irrep_types) != len(irreps_in):
raise ValueError(
"cannot build equivariant LoRA adapter: input irreps are not unique."
)
if len(out_irrep_types) != len(irreps_out):
raise ValueError(
"cannot build equivariant LoRA adapter: output irreps are not unique."
)
shared_irrep_types = in_irrep_types & out_irrep_types
if not shared_irrep_types:
raise ValueError(
"cannot build equivariant LoRA adapter: input and output irreps "
f"share no common irreps ({irreps_in!s} -> {irreps_out!s})."
)
shared_irrep_types = sorted(shared_irrep_types, key=lambda ir: (ir.l, ir.p))
return o3.Irreps([(rank, ir) for ir in shared_irrep_types])
def _build_instruction_maps(self) -> None:
"""Build maps for composing A/B adapter paths into base paths."""
# Maps input irrep index to ``(instruction_index, adapter_irrep_index,
# path_weight)`` for the input-side adapter layer.
self._A_path_by_input = {
instr.i_in: (idx, instr.i_out, instr.path_weight)
for idx, instr in enumerate(self.lora_A_layer.instructions)
}
# Maps ``(adapter_irrep_index, output_irrep_index)`` to
# ``(instruction_index, path_weight)`` for the output-side adapter layer.
self._B_path_by_adapter_output = {
(instr.i_in, instr.i_out): (idx, instr.path_weight)
for idx, instr in enumerate(self.lora_B_layer.instructions)
}
@staticmethod
def _weight_blocks_by_instruction(linear: nn.Module) -> dict[int, torch.Tensor]:
"""Reshape flat ``linear.weight`` into per-instruction blocks.
Parameters
----------
linear : nn.Module
An ``e3nn.o3.Linear`` instance.
Returns
-------
dict[int, torch.Tensor]
Instruction index to weight matrix with shape ``instr.path_shape``.
"""
blocks = {}
offset = 0
for idx, instr in enumerate(linear.instructions):
# e3nn stores all instruction weights in one flat vector. For
# o3.Linear, each instruction weight mixes multiplicity channels
# for one matching irrep path.
size = instr.path_shape[0] * instr.path_shape[1]
blocks[idx] = linear.weight[offset : offset + size].reshape(
instr.path_shape
)
offset += size
return blocks
def _merged_weight(self) -> torch.Tensor:
"""Return base plus LoRA delta in flat ``o3.Linear.weight`` layout."""
base_blocks = self._weight_blocks_by_instruction(self.base_layer)
lora_A_blocks = self._weight_blocks_by_instruction(self.lora_A_layer)
lora_B_blocks = self._weight_blocks_by_instruction(self.lora_B_layer)
merged_blocks = []
for base_idx, base_instr in enumerate(self.base_layer.instructions):
if base_instr.i_in not in self._A_path_by_input:
merged_blocks.append(base_blocks[base_idx])
continue
A_idx, i_mid, path_weight_A = self._A_path_by_input[base_instr.i_in]
B_key = (i_mid, base_instr.i_out)
if B_key not in self._B_path_by_adapter_output:
merged_blocks.append(base_blocks[base_idx])
continue
B_idx, path_weight_B = self._B_path_by_adapter_output[B_key]
ratio = path_weight_A * path_weight_B / base_instr.path_weight
delta = lora_A_blocks[A_idx] @ lora_B_blocks[B_idx]
merged_blocks.append(base_blocks[base_idx] + self.scaling * ratio * delta)
return torch.cat([block.flatten() for block in merged_blocks])
def forward(self, x: torch.Tensor, *args: object, **kwargs: object) -> torch.Tensor:
"""Run the frozen base layer plus the equivariant LoRA residual."""
out = self.base_layer(x, *args, **kwargs)
if self.enabled:
out = (
out
+ self.lora_B_layer(self.lora_A_layer(self.lora_dropout(x)))
* self.scaling
)
return out
@torch.no_grad()
def merge_into_base(self) -> None:
"""Fold the LoRA delta into the frozen base layer."""
self.base_layer.weight.copy_(self._merged_weight())
class CuEquivarianceLoRALinear(nn.Module, LoRALayer):
"""LoRA layer for cuEquivariance linear layers.
Models converted to cuEquivariance typically replace ``e3nn.o3.Linear``
modules with ``cuequivariance_torch.operations.linear.Linear``. This
wrapper keeps the same equivariant LoRA idea as
:class:`EquivariantLoRALinear`, but builds the adapter path from cuEq
linear layers so the converted module tree stays on the cuEq backend.
Notes
-----
cuEq linear modules do not expose e3nn's instruction table. This wrapper
reconstructs the equivalent per-irrep packed-weight block map from the cuEq
irreps and segmented polynomial path coefficients when merging adapters.
"""
mergeable = True
def __init__(
self,
base_layer: nn.Module,
rank: int,
alpha: float,
dropout: float = 0.0,
init: _peft.LoRAInit = "default",
**kwargs: Any,
) -> None:
nn.Module.__init__(self)
linear_cls = _cueq_linear_cls()
if not isinstance(base_layer, linear_cls):
raise TypeError(
"CuEquivarianceLoRALinear can only wrap "
"cuequivariance_torch.operations.linear.Linear layers."
)
if dropout != 0.0:
raise ValueError(
"CuEquivarianceLoRALinear does not support nonzero dropout."
)
if not getattr(base_layer, "internal_weights", False):
raise ValueError(
"CuEquivarianceLoRALinear requires cuEq Linear layers with "
"internal weights."
)
if not getattr(base_layer, "shared_weights", False):
raise ValueError(
"CuEquivarianceLoRALinear requires shared-weight cuEq Linear layers."
)
if int(getattr(base_layer, "weight_classes", 1)) != 1:
raise ValueError(
"CuEquivarianceLoRALinear does not support cuEq Linear layers "
"with multiple weight classes."
)
self.base_layer = base_layer
for param in self.base_layer.parameters():
param.requires_grad = False
self.enabled = True
self.rank = int(rank)
self.alpha = float(alpha)
self.scaling = self.alpha / self.rank
self.irreps_in = base_layer.irreps_in
self.irreps_out = base_layer.irreps_out
self.adapter_irreps = self._build_adapter_irreps(
self.irreps_in, self.irreps_out, self.rank
)
ref_weight = base_layer.weight
layout_in = getattr(base_layer.transpose_in, "source", None)
layout_out = getattr(base_layer.transpose_out, "target", None)
adapter_layout = layout_out or layout_in
self.lora_A_layer = linear_cls(
self.irreps_in,
self.adapter_irreps,
internal_weights=True,
shared_weights=True,
layout_in=layout_in,
layout_out=adapter_layout,
device=ref_weight.device,
dtype=ref_weight.dtype,
)
self.lora_B_layer = linear_cls(
self.adapter_irreps,
self.irreps_out,
internal_weights=True,
shared_weights=True,
layout_in=adapter_layout,
layout_out=layout_out,
device=ref_weight.device,
dtype=ref_weight.dtype,
)
self.lora_dropout = nn.Identity()
with torch.no_grad():
if callable(init):
init(self.lora_A_layer.weight)
elif init == "default":
self.lora_A_layer.weight.normal_(mean=0.0, std=1e-3)
else:
raise ValueError(
f"unknown init strategy {init!r}; use 'default' or a callable."
)
self.lora_B_layer.weight.zero_()
@property
def lora_A(self) -> nn.Parameter:
"""Return the input-side cuEq LoRA parameter."""
return self.lora_A_layer.weight
@property
def lora_B(self) -> nn.Parameter:
"""Return the output-side cuEq LoRA parameter."""
return self.lora_B_layer.weight
@classmethod
def is_compatible(cls, base_layer: nn.Module) -> bool:
"""Return whether ``base_layer`` can use a cuEq LoRA residual."""
try:
cls._build_adapter_irreps(
base_layer.irreps_in,
base_layer.irreps_out,
rank=1,
)
except (AttributeError, ValueError):
return False
return (
bool(getattr(base_layer, "internal_weights", False))
and bool(getattr(base_layer, "shared_weights", False))
and int(getattr(base_layer, "weight_classes", 1)) == 1
)
@staticmethod
def _build_adapter_irreps(
irreps_in: object, irreps_out: object, rank: int
) -> object:
"""Build cuEq adapter irreps from shared input/output irreps."""
try:
import cuequivariance as cue
except ImportError as exc:
raise ImportError("cuEquivariance LoRA requires cuequivariance.") from exc
irreps_in = cue.Irreps(irreps_in)
irreps_out = cue.Irreps(irreps_out)
if irreps_in.irrep_class != irreps_out.irrep_class:
raise ValueError(
"cannot build cuEq LoRA adapter for different irrep classes "
f"({irreps_in.irrep_class} vs {irreps_out.irrep_class})."
)
in_irrep_types = {mul_ir.ir for mul_ir in irreps_in}
out_irrep_types = {mul_ir.ir for mul_ir in irreps_out}
if len(in_irrep_types) != len(irreps_in):
raise ValueError(
"cannot build cuEq LoRA adapter: input irreps are not unique."
)
if len(out_irrep_types) != len(irreps_out):
raise ValueError(
"cannot build cuEq LoRA adapter: output irreps are not unique."
)
shared_irrep_types = in_irrep_types & out_irrep_types
if not shared_irrep_types:
raise ValueError(
"cannot build cuEq LoRA adapter: input and output irreps share "
f"no common irreps ({irreps_in!s} -> {irreps_out!s})."
)
shared_irrep_types = sorted(
shared_irrep_types,
key=lambda ir: (getattr(ir, "l", 0), getattr(ir, "p", 0), str(ir)),
)
return cue.Irreps(
irreps_in.irrep_class,
[(rank, ir) for ir in shared_irrep_types],
)
def forward(
self,
x: torch.Tensor,
*args: object,
**kwargs: object,
) -> torch.Tensor:
"""Run the frozen cuEq base layer plus the equivariant LoRA residual."""
out = self.base_layer(x, *args, **kwargs)
if self.enabled:
out = out + self.lora_B_layer(self.lora_A_layer(x)) * self.scaling
return out
@staticmethod
def _linear_path_coefficient(linear: nn.Module, path_index: int) -> float:
"""Return the cuEq segmented-polynomial coefficient for a linear path."""
try:
graph = linear.f.m.graphs[0].graph
coefficient = getattr(graph, f"c{path_index}")
except (AttributeError, IndexError) as exc:
raise RuntimeError(
"Cannot inspect cuEq Linear path coefficients for LoRA merge."
) from exc
return float(coefficient)
@classmethod
def _weight_blocks_by_irrep(
cls, linear: nn.Module
) -> dict[object, tuple[torch.Tensor, int, int, float]]:
"""Return cuEq linear weight blocks keyed by irrep.
The cuEq descriptor packs one ``(mul_in, mul_out)`` weight block for
each matching input/output irrep pair in descriptor path order.
"""
blocks: dict[object, tuple[torch.Tensor, int, int, float]] = {}
offset = 0
path_index = 0
weight = linear.weight
if getattr(weight, "ndim", None) != 2 or int(weight.shape[0]) != 1:
raise RuntimeError(
"CuEquivarianceLoRALinear merge requires a single shared cuEq "
"weight class."
)
for mul_in, ir_in in linear.irreps_in:
for mul_out, ir_out in linear.irreps_out:
if ir_in != ir_out:
continue
if ir_in in blocks:
raise RuntimeError(
"CuEquivarianceLoRALinear merge requires unique input "
"and output irreps."
)
size = int(mul_in) * int(mul_out)
blocks[ir_in] = (
weight[0, offset : offset + size].reshape(
int(mul_in), int(mul_out)
),
offset,
size,
cls._linear_path_coefficient(linear, path_index),
)
offset += size
path_index += 1
if offset != int(weight.shape[1]):
raise RuntimeError(
"cuEq Linear packed weight layout did not match the expected "
"per-irrep block sizes."
)
return blocks
def _merged_weight(self) -> torch.Tensor:
"""Return base plus LoRA delta in cuEq flat packed-weight layout."""
merged = self.base_layer.weight.detach().clone()
base_blocks = self._weight_blocks_by_irrep(self.base_layer)
lora_A_blocks = self._weight_blocks_by_irrep(self.lora_A_layer)
lora_B_blocks = self._weight_blocks_by_irrep(self.lora_B_layer)
for ir, (base_block, offset, size, base_coeff) in base_blocks.items():
if ir not in lora_A_blocks or ir not in lora_B_blocks:
continue
A_block, _A_offset, _A_size, A_coeff = lora_A_blocks[ir]
B_block, _B_offset, _B_size, B_coeff = lora_B_blocks[ir]
delta = (A_block @ B_block) * (
self.scaling * A_coeff * B_coeff / base_coeff
)
merged[0, offset : offset + size].copy_((base_block + delta).reshape(-1))
return merged
@torch.no_grad()
def merge_into_base(self) -> None:
"""Fold the LoRA delta into the frozen cuEq base layer."""
self.base_layer.weight.copy_(self._merged_weight())
class E3NNFullyConnectedLoRALayer(nn.Module, LoRALayer):
"""LoRA layer for fully connected scalar MLP layers ``e3nn.nn._fc._Layer``.
These internal layers store weights in ``(in_features, out_features)``
layout, so the LoRA delta is ``A @ B`` rather than the ``torch.nn.Linear``
``B @ A`` convention.
Parameters
----------
base_layer : nn.Module
An ``e3nn.nn._fc._Layer`` instance to wrap.
rank : int
LoRA rank.
alpha : float
LoRA scaling numerator.
dropout : float, optional
Adapter dropout probability. Defaults to ``0.0``.
init : _peft.LoRAInit, optional
Initialization for the input-side adapter weights (``lora_A``).
``"default"`` uses ``kaiming_uniform_(a=√5)``; pass a callable
``(tensor) -> None`` for a custom in-place scheme. ``lora_B`` is
always zero so the adapter residual starts at identity.
**kwargs : Any
Additional keyword arguments.
Attributes
----------
mergeable : bool
Required by ``LoRALayer``; whether this adapter supports merging into the
wrapped base layer.
enabled : bool
Required by ``LoRALayer``; whether the LoRA residual is active during
forward passes.
lora_A : nn.Parameter
Required by ``LoRALayer``; Input-side low-rank factor with shape ``(in_features, rank)``.
lora_B : nn.Parameter
Required by ``LoRALayer``; Output-side low-rank factor with shape ``(rank, out_features)``.
"""
mergeable = True
def __init__(
self,
base_layer: nn.Module,
rank: int,
alpha: float,
dropout: float = 0.0,
init: _peft.LoRAInit = "default",
**kwargs: Any,
) -> None:
nn.Module.__init__(self)
_linear_cls, fc_layer_cls, _o3, _dropout_cls = _import_e3nn()
if not isinstance(base_layer, fc_layer_cls):
raise TypeError(
"E3NNFullyConnectedLoRALayer can only wrap e3nn.nn._fc._Layer."
)
if not hasattr(base_layer, "weight"):
raise TypeError("e3nn fully connected layer has no weight parameter.")
if dropout != 0.0:
raise ValueError(
"E3NNFullyConnectedLoRALayer does not support nonzero dropout "
"because e3nn fully connected LoRA must be applied in weight "
"space to match merge_into_base."
)
self.base_layer = base_layer
for param in self.base_layer.parameters():
param.requires_grad = False
weight = base_layer.weight
in_features, out_features = int(weight.shape[0]), int(weight.shape[1])
self._make_lora_params(
in_features,
out_features,
weight,
int(rank),
float(alpha),
dropout,
init,
)
@classmethod
def is_compatible(cls, base_layer: nn.Module) -> bool:
"""Return whether ``base_layer`` exposes a matrix weight."""
try:
_linear_cls, fc_layer_cls, _o3, _dropout_cls = _import_e3nn()
except ImportError:
return False
return (
isinstance(base_layer, fc_layer_cls)
and getattr(getattr(base_layer, "weight", None), "ndim", None) == 2
)
def _merged_weight(self) -> torch.Tensor:
"""Return the base weight plus the low-rank adapter delta."""
return self.base_layer.weight + (self.lora_A @ self.lora_B) * self.scaling
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Run the frozen base layer plus the scalar LoRA residual."""
if not self.enabled:
return self.base_layer(x)
# e3nn _Layer normalizes ``weight`` inside forward. Applying the LoRA
# delta as an output residual would skip that normalization and would
# not match ``merge_into_base``.
# ``functional_call`` substitutes the effective weight for this call
# without mutating the frozen base layer parameter.
return functional_call(self.base_layer, {"weight": self._merged_weight()}, (x,))
@torch.no_grad()
def merge_into_base(self) -> None:
"""Fold the LoRA delta into the frozen base layer."""
self.base_layer.weight.add_((self.lora_A @ self.lora_B) * self.scaling)
_BUILTIN_LORA_WRAPPER_FACTORIES = (
(
_cueq_linear_cls,
CuEquivarianceLoRALinear,
),
(
partial(_e3nn_layer_cls, "linear"),
EquivariantLoRALinear,
),
(
partial(_e3nn_layer_cls, "fully_connected"),
E3NNFullyConnectedLoRALayer,
),
)
def _register_lora_wrappers(
registrations: LoRAWrapperRegistrations = (),
*,
temporary: bool,
) -> None:
"""Register available built-in wrappers followed by custom wrappers."""
resolved: list[tuple[LoRAWrappableLayer, LoRAWrapper]] = []
for layer_cls_factory, wrapper_cls in _BUILTIN_LORA_WRAPPER_FACTORIES:
try:
layer_cls = layer_cls_factory()
except ImportError as exc:
warnings.warn(
f"Skipping built-in LoRA wrapper {wrapper_cls.__name__}: {exc}",
UserWarning,
stacklevel=2,
)
continue
resolved.append((layer_cls, wrapper_cls))
for layer_cls, wrapper_cls in (*resolved, *registrations):
existing = _peft._LORA_WRAPPERS.get(layer_cls)
if existing is wrapper_cls:
continue
if existing is not None:
action = "Temporarily overriding" if temporary else "Overriding"
warnings.warn(
f"{action} LoRA wrapper for "
f"{layer_cls.__module__}.{layer_cls.__qualname__}: "
f"{existing.__module__}.{existing.__qualname__} -> "
f"{wrapper_cls.__module__}.{wrapper_cls.__qualname__}.",
UserWarning,
stacklevel=3,
)
_peft.register_lora_wrapper(layer_cls, wrapper_cls)
@contextmanager
def _temporary_lora_wrapper_registrations(
registrations: LoRAWrapperRegistrations,
) -> Iterator[None]:
"""Install wrappers temporarily and restore the prior registry afterward."""
previous = dict(_peft._LORA_WRAPPERS)
try:
_register_lora_wrappers(registrations, temporary=True)
yield
finally:
_peft._LORA_WRAPPERS.clear()
_peft._LORA_WRAPPERS.update(previous)
[docs]
def available_lora_wrappers() -> LoRAWrapperRegistrations:
"""Return registered (layer, LoRA-wrapper) pairs."""
_register_lora_wrappers(temporary=False)
return tuple(_peft._LORA_WRAPPERS.items())