Source code for tensorrt_edgellm.model

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Auto-dispatch model factory and parameter utilities.

``AutoModel.from_pretrained`` reads a checkpoint config, picks the right model
class, constructs it, and loads weights — the primary entry point for callers.

Custom model classes can be registered via :func:`register_model` to override
the default :class:`~models.default.modeling_default.CausalLM` for a given
``model_type`` string.
"""

import dataclasses
import itertools
import logging
import os
from typing import Callable, Dict, Set, Tuple, Type

import torch
import torch.nn as nn

from .checkpoint.checkpoint_utils import load_checkpoint_config_dicts
from .checkpoint.loader import iter_checkpoint_keys, load_weights
from .config import (_EAGLE_CANDIDATE_HEAD_KEY_PREFIXES,
                     _EAGLE_SKIPPED_CHECKPOINT_KEYS, QUANT_FP16,
                     QUANT_INT4_GPTQ, QUANT_INT8_GPTQ, ModelConfig,
                     _eagle_draft_feature_keys,
                     _is_gemma4_assistant_model_type, _is_gemma4_model_type,
                     _parse_eagle_target_layer_ids, make_dflash2_draft_config,
                     make_dflash_draft_config, make_domino_draft_config,
                     make_dspark_draft_config, make_jetspec_draft_config,
                     make_lilicorr_draft_config, make_mtp_draft_config,
                     module_quant_group_size, module_quant_type)
from .dflash import (DFlashVersion, dense_last_layer_modules,
                     resolve_dflash_contract)

__all__ = [
    "AutoModel", "load_model_config", "register_attention_scale_default",
    "register_model", "standard_attention_scale", "dtype_summary",
    "param_count"
]

AttentionScaleDefault = Callable[[int], float]
_MODEL_REGISTRY: Dict[str, Type[nn.Module]] = {}
_ATTENTION_SCALE_DEFAULT_REGISTRY: Dict[str, AttentionScaleDefault] = {}


def _instantiate_model(model_class: Type[nn.Module],
                       config: ModelConfig,
                       device: str,
                       low_cpu_mem_usage: bool = False) -> nn.Module:
    if low_cpu_mem_usage:
        if device != "cpu":
            raise ValueError("low_cpu_mem_usage requires device='cpu'")
        with torch.device("meta"):
            return model_class(config)

    model = model_class(config)
    model.to(device)
    return model


def _materialize_checkpoint_defaults(model: nn.Module, device: str) -> None:
    materialize = getattr(model, "materialize_checkpoint_defaults", None)
    if callable(materialize):
        materialize(device)


# Quantization-scale buffers that model classes register with a safe ``ones``
# default but that not every checkpoint variant carries: official
# ``nvidia/*-FP8`` / ``*-NVFP4`` checkpoints omit the attention ``q_scale``
# (ModelOpt keeps the legacy ``qScale=1.0`` convention), and INT4-AWQ exports
# omit ``pre_quant_scale`` on the resmoothed q/k/v/gate/up projections. The
# normal (non-meta) load path simply keeps the registered ``ones`` default when
# the tensor is absent; under ``low_cpu_mem_usage`` that default lands on the
# meta device, so restore it explicitly before the strict materialization scan.
_OPTIONAL_SCALE_DEFAULT_BUFFERS = frozenset(
    ("q_scale", "k_scale", "v_scale", "pre_quant_scale"))


def _materialize_optional_scale_buffers(model: nn.Module, device: str) -> None:
    """Fill in optional ``ones``-default scale buffers left on the meta device.

    Mirrors the non-``low_cpu_mem_usage`` load path, where a checkpoint that
    omits one of these buffers keeps the module's registered ``ones`` default.
    Restricted to the known-optional scale buffer names so a genuinely missing
    required tensor (e.g. a weight) still trips the strict meta-tensor scan.
    """
    for module in model.modules():
        for name, buf in module.named_buffers(recurse=False):
            if (name in _OPTIONAL_SCALE_DEFAULT_BUFFERS and buf is not None
                    and buf.device.type == "meta"):
                module.register_buffer(
                    name, torch.ones(buf.shape, dtype=buf.dtype,
                                     device=device))


def standard_attention_scale(head_dim: int) -> float:
    return 1.0 / (float(head_dim)**0.5)


_MTP_BASE_MODEL_TYPES = frozenset({
    "qwen3",
    "qwen3_5_text",
    "qwen3_5_moe",
    "qwen3_5_moe_text",
    "qwen3_omni_next_text_moe",
    "qwen4_exp_text",
})
_MTP_DRAFT_MODEL_TYPES = frozenset({
    "qwen3",
    "qwen3_5_text",
    "qwen3_5_moe_text",
    "qwen3_omni_next_text_moe",
    "qwen4_exp_text",
})


def _is_mtp_base_supported(model_type: str) -> bool:
    return model_type in _MTP_BASE_MODEL_TYPES


def _is_mtp_draft_supported(model_type: str) -> bool:
    return model_type in _MTP_DRAFT_MODEL_TYPES


def _clamp_target_layer_ids_unique(layers: list,
                                   num_decoder_layers: int) -> list:
    """Clamp target_layer_ids into [0, num_decoder_layers), keeping them unique.

    Used for EAGLE3, whose runtime cross-check (validateEagleConfig) rejects
    duplicate conditioning layers -- unlike DFlash/MTP, a plain min() clamp
    isn't enough once truncation collapses two indices onto the same layer.
    Collisions are nudged to the nearest free slot instead.
    """
    used = set()
    result = []
    for index in layers:
        candidate = max(0, min(index, num_decoder_layers - 1))
        while candidate in used and candidate < num_decoder_layers - 1:
            candidate += 1
        while candidate in used and candidate > 0:
            candidate -= 1
        used.add(candidate)
        result.append(candidate)
    return result


def _inherit_dspark_base_metadata(config: ModelConfig,
                                  dspark_draft_dir: "str | None") -> None:
    if not dspark_draft_dir:
        return

    draft_cfg_path = os.path.join(dspark_draft_dir, "config.json")
    if not os.path.isfile(draft_cfg_path):
        raise FileNotFoundError(
            f"DSpark draft config does not exist: {draft_cfg_path}")
    _, draft_cfg = load_checkpoint_config_dicts(dspark_draft_dir)
    dspark_cfg = (draft_cfg.get("dspark_config")
                  or draft_cfg.get("dflash_config") or {})

    target_layer_ids = (dspark_cfg.get("target_layer_ids")
                        or draft_cfg.get("target_layer_ids")
                        or config.dspark_target_layer_ids)
    config.dspark_target_layer_ids = list(target_layer_ids)
    config.dspark_block_size = int(
        dspark_cfg.get("block_size",
                       draft_cfg.get("block_size", config.dspark_block_size)))
    default_mask_token_id = (4 if str(draft_cfg.get(
        "model_type", "")).startswith("gemma4") else
                             config.dspark_mask_token_id)
    config.dspark_mask_token_id = int(
        dspark_cfg.get("mask_token_id",
                       draft_cfg.get("mask_token_id", default_mask_token_id)))
    config.dspark_enable_confidence_head = bool(
        dspark_cfg.get(
            "enable_confidence_head",
            draft_cfg.get("enable_confidence_head",
                          config.dspark_enable_confidence_head)))
    config.dspark_confidence_head_with_markov = bool(
        dspark_cfg.get(
            "confidence_head_with_markov",
            draft_cfg.get("confidence_head_with_markov",
                          config.dspark_confidence_head_with_markov)))
    config.dspark_markov_head_type = str(
        dspark_cfg.get(
            "markov_head_type",
            draft_cfg.get("markov_head_type", config.dspark_markov_head_type)))
    config.dspark_markov_rank = int(
        dspark_cfg.get("markov_rank",
                       draft_cfg.get("markov_rank",
                                     config.dspark_markov_rank)))
    config.dspark_sample_from_anchor = bool(
        dspark_cfg.get(
            "sample_from_anchor",
            draft_cfg.get("sample_from_anchor",
                          config.dspark_sample_from_anchor)))


def _inherit_eagle_base_layer_ids(config: ModelConfig,
                                  eagle_draft_dir: str) -> None:
    """Set an EAGLE base's aux hidden-state layers from its paired draft."""
    draft_root, draft_cfg = load_checkpoint_config_dicts(eagle_draft_dir)
    if _eagle_draft_feature_keys(draft_cfg):
        # Drafts with fc_norm, norm_output or a candidate head name their aux
        # layers as HF hidden-state indices, which Gemma4 bases do not read.
        if _is_gemma4_model_type(config.model_type):
            raise ValueError(
                "A Gemma4 EAGLE base reads draft aux layers as decoder-layer "
                f"outputs; the draft at {eagle_draft_dir} uses fc_norm, "
                "norm_output or a candidate head, whose aux layer IDs are HF "
                "hidden-state indices.")
        layer_ids = _parse_eagle_target_layer_ids(
            draft_cfg, num_layers=config.num_hidden_layers)
        if not layer_ids and draft_cfg.get("eagle_config") is not None:
            raise ValueError(
                f"EAGLE draft at {eagle_draft_dir} declares eagle_config but "
                "no aux hidden-state layer IDs.")
        config.eagle3_target_layer_ids = layer_ids
        return
    config.eagle3_target_layer_ids = list(
        draft_cfg.get("target_layer_ids", []) or [])
    if (not config.eagle3_target_layer_ids
            and draft_root.get("speculators_model_type") == "eagle3"):
        config.eagle3_target_layer_ids = [
            2, config.num_hidden_layers // 2, config.num_hidden_layers - 3
        ]


[docs] def register_model(model_type: str, model_class: Type[nn.Module], default_attention_scale: AttentionScaleDefault) -> None: """Register *model_class* as the handler for *model_type*. When :meth:`AutoModel.from_pretrained` encounters a checkpoint whose ``model_type`` field equals *model_type*, it instantiates *model_class* instead of the built-in :class:`~models.default.modeling_default.CausalLM`. Args: model_type: Value of ``model_type`` in the checkpoint ``config.json``. model_class: ``nn.Module`` subclass; must accept a single :class:`~config.ModelConfig` as its constructor argument. default_attention_scale: Function returning this family's default for a given attention head dimension. """ _MODEL_REGISTRY[model_type] = model_class _ATTENTION_SCALE_DEFAULT_REGISTRY[model_type] = default_attention_scale
def register_attention_scale_default( model_type: str, default_attention_scale: AttentionScaleDefault) -> None: """Register a default for a model type with special factory dispatch.""" _ATTENTION_SCALE_DEFAULT_REGISTRY[model_type] = default_attention_scale def load_model_config(model_dir: str) -> ModelConfig: """Load ModelConfig using the default declared by its model family.""" default_attention_scale = _default_attention_scale_for_model_dir(model_dir) return ModelConfig.from_pretrained(model_dir, default_attention_scale) def _default_attention_scale_for_model_dir( model_dir: str) -> AttentionScaleDefault: """Return the attention-scale default registered by checkpoint model_type.""" root, llm_dict = load_checkpoint_config_dicts(model_dir) default_attention_scale = standard_attention_scale for config in (root, llm_dict): model_type = config.get("model_type") if model_type in _ATTENTION_SCALE_DEFAULT_REGISTRY: default_attention_scale = _ATTENTION_SCALE_DEFAULT_REGISTRY[ model_type] break return default_attention_scale
[docs] class AutoModel: """HuggingFace-style factory that dispatches on ``model_type``."""
[docs] @classmethod def from_pretrained(cls, model_dir: str, device: str = "cpu", key_remap=None, key_prefix: "str | None" = None, eagle_base: bool = False, eagle_draft_dir: "str | None" = None, reduced_vocab_dir: "str | None" = None, mtp_base: bool = False, mtp_draft: bool = False, tp_size: int = 1, tp_rank: int = 0, dflash_base: bool = False, dflash_draft: bool = False, dflash_draft_dir: "str | None" = None, jetspec_base: bool = False, jetspec_draft: bool = False, jetspec_draft_dir: "str | None" = None, dspark_base: bool = False, dspark_draft: bool = False, dspark_draft_dir: "str | None" = None, gemma4_mtp_base: bool = False, gemma4_mtp_draft: bool = False, gemma4_kv_sharing_map: "list[dict] | None" = None, gemma4_target_kv_cache_quant: "str | None" = None, num_decoder_layers: "int | None" = None, extra_configs: "dict | None" = None, low_cpu_mem_usage: bool = False) -> nn.Module: """Construct and load a model from *model_dir*. Reads ``config.json`` via :class:`~config.ModelConfig`, looks up the model class in the registry (falling back to the built-in :class:`~models.default.modeling_default.CausalLM`), instantiates it, moves it to *device*, and loads safetensors weights. Args: model_dir: Local HF checkpoint directory. device: Target device (e.g. ``"cpu"``, ``"cuda:0"``). key_remap: Optional callable ``(key: str) -> Optional[str]``. Passed through to :func:`load_weights` for checkpoint key remapping (e.g. TTS talker ``codec_embedding`` → ``embed_tokens``). key_prefix: Explicit checkpoint key prefix to strip (e.g. ``"talker."``). Passed through to :func:`load_weights`. eagle_base: When True, export as EAGLE3 base model with extra tree-attention inputs and hidden_states output. eagle_draft_dir: Optional EAGLE3 draft checkpoint directory. Gemma4 EAGLE3 uses it to select the draft-trained target hidden layers for base hidden_states output. reduced_vocab_dir: Optional directory containing ``vocab_map.safetensors``. mtp_base: When True, export the standard Qwen3.5 text model as the MTP base variant. mtp_draft: When True, build the dedicated Qwen3.5 dense MTP draft model from the base checkpoint config. tp_size: Tensor-parallel world size. When >1 the config is reduced to per-rank shapes via :meth:`ModelConfig.for_rank`, and weights are sharded on assignment. Default 1 = no TP. tp_rank: This rank's index in [0, tp_size). dflash_base: When True, export as DFlash base model. dflash_draft: When True, build the DFlash draft model. dflash_draft_dir: Path to the DFlash draft checkpoint directory. jetspec_base: When True, export as JetSpec base model. jetspec_draft: When True, build the JetSpec draft model. jetspec_draft_dir: Path to the JetSpec draft checkpoint directory. dspark_base: When True, export as DSpark base model. dspark_draft: When True, build the DSpark draft backbone model. dspark_draft_dir: Path to the DSpark draft checkpoint directory. gemma4_mtp_base: Export a Gemma4 target checkpoint as the base engine for paired Gemma4 MTP. gemma4_mtp_draft: Export/load a paired Gemma4 assistant checkpoint. gemma4_kv_sharing_map: Validated assistant-layer to target-layer map for Gemma4 MTP draft runtime config. gemma4_target_kv_cache_quant: Target/base KV-cache quantization mode inherited by Gemma4 MTP draft inputs that alias target KV cache. num_decoder_layers: When set, truncate the model to only the first N decoder layers (few-layer numeric validation). Supported for the plain default ``CausalLM`` path (e.g. Qwen3), registered hybrid base models, MTP base, and eagle/dflash base (their target_layer_ids are clamped into the truncated range); rejected for jetspec/dspark/gemma4-mtp and any draft variant. The checkpoint's extra-layer weights are simply skipped by the loader. low_cpu_mem_usage: Initialize model tensors on the meta device and materialize them from the checkpoint during load. Returns: Loaded ``nn.Module`` in eval mode. """ from .models.default.modeling_default import CausalLM config = load_model_config(model_dir) if extra_configs: for key, value in extra_configs.items(): setattr(config, key, value) # Qwen3-Omni Next ships both dense and sparse-MoE thinkers under the # same ``qwen3_omni_next_text`` model_type (the HF config is not # rewritten for the MoE variant). Detect MoE by ``num_experts > 0`` and # re-dispatch to the registered ``_moe`` class variant when one exists. if (getattr(config, "num_experts", 0) and f"{config.model_type}_moe" in _MODEL_REGISTRY): config.model_type = f"{config.model_type}_moe" if eagle_base: config.eagle_base = True if eagle_draft_dir: _inherit_eagle_base_layer_ids(config, eagle_draft_dir) if mtp_base or config.mtp_base: config.mtp_base = True if gemma4_mtp_base: config.gemma4_mtp_base = True if dflash_base: config.dflash_base = True dflash_version = DFlashVersion.V1 dflash_projector = "" if config.dflash_base: if dflash_draft_dir: draft_root, draft_llm = load_checkpoint_config_dicts( dflash_draft_dir) contract = resolve_dflash_contract(draft_root, draft_llm) dflash_version = contract.version config.dflash_version = contract.version config.dflash_target_layer_ids = list( contract.target_layer_ids) config.dflash_block_size = contract.block_size config.dflash_mask_token_id = contract.mask_token_id if contract.is_domino: dflash_projector = contract.projector_type config.dflash_projector_type = contract.projector_type config.domino_gru_hidden_dim = contract.gru_hidden_dim config.domino_proj_dim = contract.proj_dim config.domino_prefix_len = contract.prefix_len config.domino_pool_size = min(contract.pool_size, config.vocab_size) config.domino_shift_label = contract.shift_label if contract.version == DFlashVersion.V2: make_v2_draft_config = (make_lilicorr_draft_config if contract.is_lilicorr else make_dflash2_draft_config) draft_config = make_v2_draft_config( dflash_draft_dir, _default_attention_scale_for_model_dir( dflash_draft_dir)) if (config.hidden_size != draft_config.hidden_size or config.vocab_size != draft_config.vocab_size): raise ValueError( "DFlash V2 base/draft hidden and vocabulary sizes must match." ) invalid = [ index for index in contract.target_layer_ids if index < 0 or index >= config.num_hidden_layers ] if invalid: raise ValueError( f"DFlash V2 target-layer IDs outside base model: {invalid}" ) config.dflash2_target_layer_ids = list( contract.target_layer_ids) config.dflash2_block_size = contract.block_size config.dflash2_mask_token_id = contract.mask_token_id config.dflash2_is_causal = contract.is_causal config.dflash2_conv_kernel_size = contract.conv_kernel_size config.dflash2_conv_group_size = contract.conv_group_size if contract.is_lilicorr: _require_unscaled_lilicorr_embedding(config) config.dflash_projector_type = contract.projector_type for field in dataclasses.fields(draft_config): if field.name.startswith("lilicorr_"): setattr(config, field.name, getattr(draft_config, field.name)) else: config.dflash2_selector_rank = contract.selector_rank config.dflash2_selector_top_k = contract.selector_top_k if not config.dflash_target_layer_ids: config.dflash_target_layer_ids = [1, 8, 15, 22, 29] elif dflash_draft and dflash_draft_dir: draft_root, draft_llm = load_checkpoint_config_dicts( dflash_draft_dir) contract = resolve_dflash_contract(draft_root, draft_llm) dflash_version = contract.version dflash_projector = contract.projector_type if jetspec_base: config.jetspec_base = True if config.jetspec_base: if not config.jetspec_target_layer_ids and jetspec_draft_dir: import json draft_cfg_path = os.path.join(jetspec_draft_dir, "config.json") if os.path.isfile(draft_cfg_path): with open(draft_cfg_path) as f: draft_cfg = json.load(f) jetspec_cfg = (draft_cfg.get("jetspec_config") or draft_cfg.get("dflash_config") or {}) config.jetspec_target_layer_ids = list( jetspec_cfg.get("target_layer_ids", draft_cfg.get("target_layer_ids", [])) or []) config.jetspec_block_size = int( jetspec_cfg.get("block_size", draft_cfg.get("block_size", 16))) config.jetspec_mask_token_id = int( jetspec_cfg.get("mask_token_id", draft_cfg.get("mask_token_id", 151669))) config.jetspec_causal_head = bool( jetspec_cfg.get("causal_head", draft_cfg.get("causal_head", True))) if not config.jetspec_target_layer_ids: raise ValueError( "jetspec_base requires JetSpec target_layer_ids; pass " "jetspec_draft_dir or set jetspec_target_layer_ids.") if not config.jetspec_causal_head: raise ValueError( "jetspec_base requires jetspec_config.causal_head=true.") config.dflash_target_layer_ids = list( config.jetspec_target_layer_ids) config.dflash_block_size = config.jetspec_block_size config.dflash_mask_token_id = config.jetspec_mask_token_id if dspark_base: config.dspark_base = True if config.dspark_base: _inherit_dspark_base_metadata(config, dspark_draft_dir) if not config.dspark_target_layer_ids: raise ValueError( "dspark_base requires DSpark target_layer_ids; pass " "dspark_draft_dir or set dspark_target_layer_ids.") if tp_size > 1: config = config.for_rank(tp_rank, tp_size) variant = _resolve_model_variant(config, eagle_base=eagle_base, mtp_base=config.mtp_base, mtp_draft=mtp_draft, dflash_base=config.dflash_base, dflash_draft=dflash_draft, dflash_version=dflash_version, dflash_projector=dflash_projector, jetspec_base=config.jetspec_base, jetspec_draft=jetspec_draft, dspark_base=config.dspark_base, dspark_draft=dspark_draft, gemma4_mtp_base=gemma4_mtp_base, gemma4_mtp_draft=gemma4_mtp_draft) # EAGLE3 draft: auto-detect from draft_vocab_size if variant == "eagle3_draft": from .models.eagle3.modeling_eagle3_draft import Eagle3DraftModel model_class = Eagle3DraftModel # Set up key remapping: midlayer -> layers.0, skip t2d if key_remap is None: key_remap = _eagle3_key_remap elif variant == "eagle4_draft": from .models.eagle4 import Eagle4DraftModel model_class = Eagle4DraftModel if key_remap is None and config.draft_candidate_count > 0: key_remap = _eagle4_key_remap elif key_remap is None: key_remap = _eagle4_dense_head_key_remap elif variant == "mtp_draft": is_nemotron_h_mtp = config.is_nemotron_h # TODO: support other model types if not (is_nemotron_h_mtp or _is_mtp_draft_supported(config.model_type)): raise NotImplementedError( "MTP draft is only supported for qwen3_5_text / " "qwen3_5_moe_text / qwen3_omni_next_text_moe / qwen3 / " "qwen4_exp_text / Nemotron-H checkpoints; got " f"{config.model_type!r}.") draft_model_type = config.model_type tie_word_embeddings = config.tie_word_embeddings config = make_mtp_draft_config(config) if is_nemotron_h_mtp: from .models.nemotron_h import NemotronHMtpDraftModel model_class = NemotronHMtpDraftModel elif draft_model_type == "qwen3_omni_next_text_moe": from .models.qwen3_omni_next import \ Qwen3OmniNextMoeMtpDraftModel model_class = Qwen3OmniNextMoeMtpDraftModel elif draft_model_type == "qwen3_5_moe_text": from .models.qwen3_5_moe import Qwen3_5MoeMtpDraftModel model_class = Qwen3_5MoeMtpDraftModel elif draft_model_type == "qwen3": from .models.mtp import Qwen3MtpDraftModel model_class = Qwen3MtpDraftModel elif draft_model_type == "qwen4_exp_text": from .models.qwen4_exp import Qwen4ExpMtpDraftModel model_class = Qwen4ExpMtpDraftModel else: from .models.qwen3_5 import Qwen3_5MtpDraftModel model_class = Qwen3_5MtpDraftModel if key_remap is None: # Only borrow the base model's lm_head when the checkpoint # ships no dedicated ``mtp.lm_head.*`` tensors. share_base_lm_head = not any( key.startswith("mtp.lm_head.") for key in iter_checkpoint_keys(model_dir)) def key_remap(key): return _mtp_key_remap( key, tie_word_embeddings=tie_word_embeddings, share_base_lm_head=share_base_lm_head) elif variant == "dflash_draft": if dflash_draft_dir is None: raise ValueError( "dflash_draft requires dflash_draft_dir to be set.") from .models.dflash.modeling_dflash_draft import DFlashDraftModel base_config = config base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( dflash_draft_dir) config = make_dflash_draft_config( dflash_draft_dir, _default_attention_scale_for_model_dir(dflash_draft_dir), target_vocab_size=base_config.vocab_size, quantize_w4a16_activations=True) config.dflash_overflow_fp32 = ( base_config.model_type.startswith("gemma4") and config.model_type == "qwen3") if config.dflash_overflow_fp32: config.quant.excluded = list( config.quant.excluded) + dense_last_layer_modules( config.num_hidden_layers) if (base_config.model_type == "nemotron_h" or any(key == "fc.weight_scale" for key in iter_checkpoint_keys(dflash_draft_dir))): # A quantized fc runs at the checkpoint's native precision, like # a Nemotron-3.5 target (hidden states far inside FP16), rather # than the dense-FP16 + FP32 projection that guards Qwen3-8B # (target-hidden ~abs 2e4). config.dflash_fc_native_precision = True config.quant.excluded = [ e for e in config.quant.excluded if e != "fc" ] if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = DFlashDraftModel model_dir = dflash_draft_dir if key_remap is None: key_remap = (_muse_glimmer_dflash_key_remap if config.model_type == "muse_glimmer_assistant" else _dflash_key_remap) elif variant == "dflash2_draft": if dflash_draft_dir is None: raise ValueError( "DFlash V2 draft requires dflash_draft_dir to be set.") from .models.dflash2 import DFlash2DraftModel base_config = config base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( dflash_draft_dir) config = make_dflash2_draft_config( dflash_draft_dir, _default_attention_scale_for_model_dir(dflash_draft_dir)) if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = DFlash2DraftModel model_dir = dflash_draft_dir if key_remap is None: key_remap = _dflash_key_remap elif variant == "lilicorr_draft": if dflash_draft_dir is None: raise ValueError( "LiLiCorr draft requires dflash_draft_dir to be set.") if reduced_vocab_dir is not None: raise ValueError( "LiLiCorr drafts do not support reduced-vocabulary export: " "the candidate ids index the full target vocabulary and " "its runtime embedding table.") from .models.lilicorr import LiLiCorrDraftModel base_config = config _require_unscaled_lilicorr_embedding(base_config) base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( dflash_draft_dir, _dflash_key_remap) config = make_lilicorr_draft_config( dflash_draft_dir, _default_attention_scale_for_model_dir(dflash_draft_dir)) if _checkpoint_has_draft_vocab_map(dflash_draft_dir): raise ValueError( "LiLiCorr candidates require the full target vocabulary: " "the draft checkpoint ships a d2t draft-to-target map") if config.vocab_size != base_config.vocab_size: raise ValueError( "LiLiCorr draft vocabulary must equal the base vocabulary: " "its candidate ids index the base embedding table " f"(draft {config.vocab_size}, base {base_config.vocab_size})" ) # fc and the head linears stay packed NVFP4 (W4A16 GEMM). config.dflash_fc_native_precision = True if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = LiLiCorrDraftModel model_dir = dflash_draft_dir if key_remap is None: key_remap = _dflash_key_remap elif variant == "domino_draft": if dflash_draft_dir is None: raise ValueError( "Domino draft requires dflash_draft_dir to be set.") from .models.domino import DominoDraftModel base_config = config base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( dflash_draft_dir, _domino_key_remap) config = make_domino_draft_config( dflash_draft_dir, _default_attention_scale_for_model_dir(dflash_draft_dir)) if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = DominoDraftModel model_dir = dflash_draft_dir if key_remap is None: key_remap = _domino_key_remap elif variant == "jetspec_draft": if jetspec_draft_dir is None: raise ValueError( "jetspec_draft requires jetspec_draft_dir to be set.") from .models.dflash.modeling_dflash_draft import DFlashDraftModel base_config = config base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( jetspec_draft_dir) config = make_jetspec_draft_config( jetspec_draft_dir, _default_attention_scale_for_model_dir(jetspec_draft_dir)) if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = DFlashDraftModel model_dir = jetspec_draft_dir if key_remap is None: key_remap = _dflash_key_remap elif variant == "gemma4_mtp_draft": if not _is_gemma4_assistant_model_type(config.root_model_type): raise ValueError( "Gemma4 MTP draft requires a Gemma4 assistant checkpoint.") from .models.gemma4 import Gemma4AssistantForCausalLM config.gemma4_mtp_draft = True config.shares_target_kv = True config.has_own_kv_cache = False config.constant_draft_positions = True config.returns_feedback_hidden = True config.assistant_hidden_size = config.hidden_size config.kv_sharing_map = list(gemma4_kv_sharing_map or []) if gemma4_target_kv_cache_quant is not None: config.quant.kv_cache_quant = gemma4_target_kv_cache_quant model_class = Gemma4AssistantForCausalLM elif variant == "dspark_draft": if dspark_draft_dir is None: raise ValueError( "dspark_draft requires dspark_draft_dir to be set.") from .models.dspark.modeling_dspark_draft import DSparkDraftModel base_config = config base_model_dir = model_dir base_tie_word_embeddings = base_config.tie_word_embeddings draft_has_lm_head = _checkpoint_has_dflash_lm_head( dspark_draft_dir, _dspark_key_remap) config = make_dspark_draft_config( dspark_draft_dir, _default_attention_scale_for_model_dir(dspark_draft_dir)) if base_config.model_type == "nemotron_h": config.dspark_fc_native_precision = True if not draft_has_lm_head: config = _inherit_dflash_lm_head_quant(config, base_config) model_class = DSparkDraftModel model_dir = dspark_draft_dir if key_remap is None: key_remap = _dspark_key_remap else: if (variant == "mtp_base" and not config.is_nemotron_h and not _is_mtp_base_supported(config.model_type)): raise NotImplementedError( "MTP base is only supported for Qwen3.5 (text/MoE), dense " "Qwen3, Qwen4-Exp and Nemotron-H checkpoints; got " f"{config.model_type!r}.") if variant == "gemma4_mtp_base": if config.model_type not in ("gemma4", "gemma4_text", "gemma4_unified", "gemma4_unified_text"): raise ValueError( "Gemma4 MTP base requires a Gemma4 target checkpoint.") from .models.gemma4 import Gemma4ForCausalLM config.gemma4_mtp_base = True model_class = Gemma4ForCausalLM else: # DFlash base is supported for both Qwen3.5 hybrid (qwen3_5_text) and # dense Qwen3 (default CausalLM). Dense models use the Transformer's # dflash_target_layer_ids parameter to collect target-layer hidden states. model_class = _MODEL_REGISTRY.get(config.model_type, CausalLM) if (key_remap is None and config.model_type == "hunyuan_v1_dense"): key_remap = _hunyuan_key_remap if (key_remap is None and str(config.model_type).startswith("muse_glimmer")): from .models.muse_glimmer import MUSE_GLIMMER_KEY_REMAP key_remap = MUSE_GLIMMER_KEY_REMAP # 4-layer numeric validation: truncate to the first N decoder # layers. The whole pipeline is config-driven (the Transformer builds # layers from ``config.num_hidden_layers`` / ``config.layer_types``, # ``onnx_export_spec`` derives the KV / recurrent / conv I/O counts from # them, the runtime config.json is written from ``model.config``, and the # loader silently skips checkpoint keys for the dropped layers), so a # single config override is sufficient and the modeling code needs no # change. This covers the plain default ``CausalLM`` path (e.g. Qwen3) # AND registered hybrid base models (Qwen3.5 linear+full / Gated DeltaNet, # Nemotron-H Mamba), which also build per-layer from ``layer_types``. # # Single-checkpoint MTP (``--mtp``) is also covered: its draft head reads # only the base's *last* hidden state, which stays well defined after # truncation. (The draft was trained against the full stack, so its # proposals are near-useless on a truncated base and acceptance collapses # to one token per round -- that costs speed, not correctness, and the # few-layer comparison is about the base model's committed state.) # # Eagle3/DFlash base: their target_layer_ids index the *full* stack, so they # are clamped back into range below. DFlash tolerates duplicates after # clamping; EAGLE3's runtime cross-check does not, so its clamp dedupes. # # jetspec/dspark/gemma4-mtp stay out of scope: each names its own target-layer # contract. The MTP/eagle/dflash *draft* is likewise excluded; only the base # is compared against the golden. if num_decoder_layers is not None: if (jetspec_base or config.jetspec_base or dspark_base or config.dspark_base or mtp_draft or dflash_draft or jetspec_draft or dspark_draft or config.is_jetspec_draft or config.is_dspark_draft or gemma4_mtp_base or config.gemma4_mtp_base or gemma4_mtp_draft or config.gemma4_mtp_draft): raise NotImplementedError( "num_decoder_layers cannot be combined with the " "jetspec/dspark/gemma4-mtp speculative-decoding variants " "(single-checkpoint MTP base, and eagle/dflash base with " "clamped target_layer_ids, are supported).") if not 1 <= num_decoder_layers <= config.num_hidden_layers: raise ValueError( f"num_decoder_layers={num_decoder_layers} out of range " f"[1, {config.num_hidden_layers}].") config.num_hidden_layers = num_decoder_layers if eagle_base or config.eagle_base: if num_decoder_layers < 3: raise ValueError( "EAGLE3 requires at least 3 decoder layers to provide " "three distinct conditioning layers; got " f"num_decoder_layers={num_decoder_layers}") config.eagle3_target_layer_ids = _clamp_target_layer_ids_unique( config.eagle3_target_layer_ids, num_decoder_layers) if dflash_base or config.dflash_base: # Duplicates after clamping only make the draft head reread the same # shallow hidden state: degraded proposals, not a broken pipeline. config.dflash_target_layer_ids = [ min(i, num_decoder_layers - 1) for i in config.dflash_target_layer_ids ] config.dflash2_target_layer_ids = [ min(i, num_decoder_layers - 1) for i in config.dflash2_target_layer_ids ] # Keep per-layer lists consistent with the truncated layer count. if config.layer_types: config.layer_types = config.layer_types[:num_decoder_layers] logging.getLogger(__name__).info( "num_decoder_layers: truncated to first %d decoder layers", num_decoder_layers) model = _instantiate_model(model_class, config, device, low_cpu_mem_usage=low_cpu_mem_usage) if variant == "lilicorr_draft": _fill_lilicorr_load_sentinels(model) pre_repack_hook = None apply_reduced_vocab_after_load = False if reduced_vocab_dir is not None: from .vocab_reduction.onnx_export import ( apply_reduced_vocab, load_reduced_vocab_map, should_apply_reduced_vocab_before_repacking) vocab_map = load_reduced_vocab_map(reduced_vocab_dir, vocab_size=config.vocab_size, device=device) if should_apply_reduced_vocab_before_repacking(model): def _apply_pre_repack_reduced_vocab(loaded_model: nn.Module): apply_reduced_vocab(loaded_model, vocab_map) loaded_model._reduced_vocab_dir = reduced_vocab_dir pre_repack_hook = _apply_pre_repack_reduced_vocab else: apply_reduced_vocab_after_load = True if variant in ("dflash_draft", "dflash2_draft", "lilicorr_draft", "domino_draft", "jetspec_draft", "dspark_draft") and not draft_has_lm_head: next_pre_repack_hook = pre_repack_hook def _load_pre_repack_dflash_lm_head(loaded_model: nn.Module): _load_dflash_lm_head( loaded_model, base_model_dir, device, tie_word_embeddings=base_tie_word_embeddings) if next_pre_repack_hook is not None: next_pre_repack_hook(loaded_model) pre_repack_hook = _load_pre_repack_dflash_lm_head if config.ple_enabled: checkpoint_key_remap = key_remap def _skip_runtime_ple(key): mapped = checkpoint_key_remap( key) if checkpoint_key_remap else key if mapped in ("model.embed_tokens_per_layer.weight", "embed_tokens_per_layer.weight"): return None return mapped key_remap = _skip_runtime_ple if variant == "dflash_draft": next_hook = pre_repack_hook def _dequantize_packed_fp16_linears(loaded_model: nn.Module): _dequantize_packed_dense_linears(loaded_model, model_dir, device) if next_hook is not None: next_hook(loaded_model) pre_repack_hook = _dequantize_packed_fp16_linears if variant == "domino_draft": next_domino_hook = pre_repack_hook def _load_pre_repack_domino_pool(loaded_model: nn.Module): _load_domino_pool_weights(loaded_model, model_dir, device) if next_domino_hook is not None: next_domino_hook(loaded_model) pre_repack_hook = _load_pre_repack_domino_pool if variant == "lilicorr_draft": next_lilicorr_hook = pre_repack_hook lilicorr_key_remap = key_remap def _finish_pre_repack_lilicorr(loaded_model: nn.Module): if next_lilicorr_hook is not None: next_lilicorr_hook(loaded_model) _finish_lilicorr_load(loaded_model, model_dir, lilicorr_key_remap) pre_repack_hook = _finish_pre_repack_lilicorr load_weights(model, model_dir, device=device, key_remap=key_remap, key_prefix=key_prefix, pre_repack_hook=pre_repack_hook, mapping=config.mapping) if low_cpu_mem_usage: _materialize_checkpoint_defaults(model, device) _materialize_optional_scale_buffers(model, device) meta_tensors = [ name for name, tensor in (*model.named_parameters(), *model.named_buffers()) if tensor.device.type == "meta" ] if meta_tensors: raise RuntimeError( "Checkpoint did not materialize model tensors: " + ", ".join(meta_tensors[:10])) if variant == "eagle4_draft": _require_eagle4_draft_tensors(model, iter_checkpoint_keys(model_dir)) refresh_router_bias = getattr(model, "refresh_fp32_router_bias", None) if callable(refresh_router_bias): refresh_router_bias() if variant in ("dflash_draft", "dflash2_draft", "lilicorr_draft", "domino_draft", "jetspec_draft", "dspark_draft"): if draft_has_lm_head: logging.getLogger(__name__).info( "%s lm_head source: draft checkpoint buffers", { "jetspec_draft": "JetSpec", "dspark_draft": "DSpark", "domino_draft": "Domino", "lilicorr_draft": "LiLiCorr", }.get(variant, "DFlash")) if apply_reduced_vocab_after_load: from .vocab_reduction.onnx_export import \ apply_reduced_vocab_from_dir apply_reduced_vocab_from_dir(model, reduced_vocab_dir) # Post-load optimisation: fuse GDN input projections for every family whose mixer is # the shared GdnMixer. if (config.model_type in ("qwen3_5_text", "qwen3_5_moe_text", "qwen4_exp_text") and not mtp_draft): from .models.qwen3_5 import fuse_gdn_input_projections fuse_gdn_input_projections(model) # Fuse attention Q/K/V projections into one packed-QKV GEMM where eligible. from .models.default.modeling_default import fuse_qkv_projections fuse_qkv_projections(model) return model
def param_count(model: nn.Module) -> int: """Return total parameter element count (trainable and frozen).""" return sum(p.numel() for p in model.parameters()) def dtype_summary(model: nn.Module) -> Dict[str, int]: """Map dtype name -> number of parameter elements.""" out: Dict[str, int] = {} for p in model.parameters(): name = str(p.dtype).replace("torch.", "") out[name] = out.get(name, 0) + p.numel() return dict(sorted(out.items(), key=lambda x: -x[1])) def _inherit_dflash_lm_head_quant(draft_config: ModelConfig, base_config: ModelConfig) -> ModelConfig: """Make an old DFlash draft build a loadable shared lm_head. Older DFlash drafts may omit ``lm_head.*`` and reuse the base output head. When that head is quantized, mirror its layout only for the draft ``lm_head`` so the fallback loader can copy the base sidecar tensors. """ lm_head_quant = module_quant_type("lm_head", base_config) if lm_head_quant == QUANT_FP16: return draft_config draft_quant = draft_config.quant base_quant = base_config.quant lm_head_group_size = module_quant_group_size("lm_head", base_config) layer_overrides = dict(draft_quant.layer_overrides) layer_overrides["lm_head"] = lm_head_quant excluded = [name for name in draft_quant.excluded if name != "lm_head"] layer_group_sizes = dict(draft_quant.layer_group_sizes) layer_group_sizes["lm_head"] = lm_head_group_size quant_updates = { "excluded": excluded, "layer_overrides": layer_overrides, "layer_group_sizes": layer_group_sizes, } if lm_head_quant in (QUANT_INT4_GPTQ, QUANT_INT8_GPTQ): quant_updates[ "gptq_zero_point_offset"] = base_quant.gptq_zero_point_offset quant = dataclasses.replace(draft_quant, **quant_updates) return dataclasses.replace(draft_config, quant=quant) # --------------------------------------------------------------------------- # EAGLE3 helpers # --------------------------------------------------------------------------- def _resolve_model_variant(config: ModelConfig, *, eagle_base: bool, mtp_base: bool, mtp_draft: bool, dflash_base: bool = False, dflash_draft: bool = False, dflash_version: DFlashVersion = DFlashVersion.V1, dflash_projector: str = "", jetspec_base: bool = False, jetspec_draft: bool = False, dspark_base: bool = False, dspark_draft: bool = False, gemma4_mtp_base: bool = False, gemma4_mtp_draft: bool = False) -> str: """Resolve the requested model variant while keeping EAGLE3 behavior intact.""" if eagle_base and mtp_base: raise ValueError("eagle_base and mtp_base cannot both be enabled.") if eagle_base and mtp_draft: raise ValueError("eagle_base and mtp_draft cannot both be enabled.") if mtp_base and mtp_draft: raise ValueError("mtp_base and mtp_draft cannot both be enabled.") if dflash_base and dflash_draft: raise ValueError( "DFlash base and draft variants cannot both be enabled.") if jetspec_base and jetspec_draft: raise ValueError( "jetspec_base and jetspec_draft cannot both be enabled.") if dspark_base and dspark_draft: raise ValueError( "dspark_base and dspark_draft cannot both be enabled.") if dflash_base and (eagle_base or mtp_base or mtp_draft or jetspec_base or jetspec_draft or dspark_base or dspark_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "dflash_base cannot be combined with other spec variants.") if dflash_draft and (eagle_base or mtp_base or mtp_draft or jetspec_base or jetspec_draft or dspark_base or dspark_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "dflash_draft cannot be combined with other spec variants.") if jetspec_base and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or dspark_base or dspark_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "jetspec_base cannot be combined with other spec variants.") if jetspec_draft and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or dspark_base or dspark_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "jetspec_draft cannot be combined with other spec variants.") if dspark_base and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or jetspec_base or jetspec_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "dspark_base cannot be combined with other spec variants.") if dspark_draft and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or jetspec_base or jetspec_draft or gemma4_mtp_base or gemma4_mtp_draft): raise ValueError( "dspark_draft cannot be combined with other spec variants.") if gemma4_mtp_base and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or jetspec_base or jetspec_draft or dspark_base or dspark_draft): raise ValueError( "gemma4_mtp_base cannot be combined with other speculative variants." ) if gemma4_mtp_draft and (eagle_base or mtp_base or mtp_draft or dflash_base or dflash_draft or jetspec_base or jetspec_draft or dspark_base or dspark_draft): raise ValueError( "gemma4_mtp_draft cannot be combined with other speculative variants." ) if gemma4_mtp_base and gemma4_mtp_draft: raise ValueError( "gemma4_mtp_base and gemma4_mtp_draft cannot both be enabled.") if config.is_eagle3_draft: if mtp_base or mtp_draft: raise ValueError( "EAGLE3 draft checkpoints cannot be loaded as Qwen3.5 MTP variants." ) if config.is_eagle4_draft: return "eagle4_draft" return "eagle3_draft" if gemma4_mtp_draft: return "gemma4_mtp_draft" if dflash_draft and dflash_projector == "domino": return "domino_draft" if dflash_draft and dflash_projector == "lilicorr": return "lilicorr_draft" if dflash_draft and dflash_version == DFlashVersion.V2: return "dflash2_draft" if dflash_draft: return "dflash_draft" if dflash_base: return "dflash_base" if jetspec_draft: return "jetspec_draft" if jetspec_base: return "jetspec_base" if dspark_draft: return "dspark_draft" if dspark_base: return "dspark_base" if mtp_draft: return "mtp_draft" if mtp_base: return "mtp_base" if gemma4_mtp_base: return "gemma4_mtp_base" if eagle_base: return "eagle_base" return "llm" def _hunyuan_key_remap(key: str) -> "str | None": """Remap HunYuan V1 checkpoint keys to the default CausalLM module tree. HunYuan names its per-head QK norms ``query_layernorm`` / ``key_layernorm`` where the default :class:`Attention` module uses ``q_norm`` / ``k_norm``. """ key = key.replace(".self_attn.query_layernorm.", ".self_attn.q_norm.") key = key.replace(".self_attn.key_layernorm.", ".self_attn.k_norm.") return key def _eagle3_key_remap(key: str) -> "str | None": """Remap EAGLE3 draft checkpoint keys. Handles all known EAGLE3 draft checkpoint variations: - ``t2d`` keys are skipped (but ``d2t`` is kept). - ``target_model.*`` keys are skipped (multi-target training artifact). - ``midlayer.*`` -> ``layers.0.*`` - ``qkv_proj.{q,k,v}_proj`` -> ``{q,k,v}_proj`` (flatten old pipeline ``EdgeLLMAttention`` wrapper nesting, used by quantized checkpoints). - ``._pre_quant_scale`` -> ``.pre_quant_scale`` (modelopt internal naming; normally stripped by ``postprocess_state_dict()`` but not by per-module export via ``_export_quantized_weight()``). """ if "t2d" in key and "d2t" not in key: return None # skip t2d if key.startswith("target_model."): return None # skip multi-target training artifact key = key.replace("midlayer.", "layers.0.") key = key.replace("qkv_proj.q_proj", "q_proj") key = key.replace("qkv_proj.k_proj", "k_proj") key = key.replace("qkv_proj.v_proj", "v_proj") key = key.replace("._pre_quant_scale", ".pre_quant_scale") return key def _eagle4_key_remap(key: str) -> "str | None": """:func:`_eagle3_key_remap` for drafts with a low-rank candidate head. ``embed_tokens.weight`` is skipped (drafts read the base embedding), and the candidate head factors load into ``candidate_head``: ``efficient_lm_head_down_proj.*`` -> ``candidate_head.down_proj.*`` and ``efficient_lm_head.*`` -> ``candidate_head.up_proj.*``. """ if key in _EAGLE_SKIPPED_CHECKPOINT_KEYS: return None for prefix, module_prefix in _EAGLE_CANDIDATE_HEAD_KEY_PREFIXES: if key.startswith(prefix): return module_prefix + key[len(prefix):] return _eagle3_key_remap(key) def _eagle4_dense_head_key_remap(key: str) -> "str | None": """:func:`_eagle4_key_remap` for drafts exported with the dense head. The low-rank candidate head factors are skipped. """ module_key = _eagle4_key_remap(key) if module_key is not None and module_key.startswith("candidate_head."): return None return module_key def _checkpoint_tensor_names(model: nn.Module) -> Tuple[Set[str], Set[str]]: """Return the model's tensor names as checkpoint names, plus unread names. NVFP4-A16 linears repack ``weight``, ``weight_scale`` and ``weight_scale_2`` into plugin buffers on load and keep no buffer for the W4A4 ``input_scale`` (the second set). """ from .models.linear import NVFP4A16Linear repacked = { "qweight": "weight", "block_scales": "weight_scale", "global_scale": "weight_scale_2", } names = set(model.state_dict()) unread = set() for module_name, module in model.named_modules(): if not isinstance(module, NVFP4A16Linear): continue prefix = f"{module_name}." if module_name else "" for buffer_name, checkpoint_name in repacked.items(): if prefix + buffer_name in names: names.discard(prefix + buffer_name) names.add(prefix + checkpoint_name) unread.add(prefix + "input_scale") return names, unread def _require_eagle4_draft_tensors(model: nn.Module, checkpoint_keys) -> None: """Raise unless checkpoint and draft-model tensors match one to one. The loader only counts skipped keys, so a misnamed or missing tensor would otherwise leave a module at its initial value. A draft model without a candidate head skips the checkpoint's candidate head factors. """ has_candidate_head = getattr(model, "candidate_head", None) is not None key_remap = (_eagle4_key_remap if has_candidate_head else _eagle4_dense_head_key_remap) remapped = {} for key in checkpoint_keys: module_key = key_remap(key) if module_key is not None: remapped[module_key] = key state = model.state_dict() names, unread = _checkpoint_tensor_names(model) unused = sorted(key for module_key, key in remapped.items() if module_key not in names and module_key not in unread) missing = names - remapped.keys() # A checkpoint may omit d2t only when the module default is already the # identity map, i.e. the draft vocabulary is the full vocabulary. d2t = state.get("d2t") if (d2t is not None and d2t.device.type != "meta" and torch.equal( d2t.reshape(-1), torch.arange(d2t.numel(), dtype=d2t.dtype, device=d2t.device))): missing.discard("d2t") missing = sorted(missing) if unused or missing: raise ValueError( "EAGLE draft checkpoint does not match the draft model. " f"Checkpoint tensors without a model tensor: {unused or 'none'}. " f"Model tensors without a checkpoint tensor: {missing or 'none'}.") def _dequantize_packed_dense_linears(model: nn.Module, model_dir: str, device: str) -> None: """Dequantize dense ``FP16Linear`` modules whose checkpoint weight is packed NVFP4. The generic loader leaves the packed ``uint8 [out, in/2]`` payload in the module; rebuild the dense weight from the packed weight, block scales and global scale, as the checkpoint-direct builder does. """ from .checkpoint.loader import _build_shard_map from .models.linear import FP16Linear from .models.ops import _nvfp4_dequantize_eager shard_map = _build_shard_map(model_dir) for name, module in model.named_modules(): if (isinstance(module, FP16Linear) and module.weight.dtype == torch.uint8): scale, global_scale = (_read_dflash_checkpoint_tensor( shard_map[key], key, device) for key in (f"{name}.weight_scale", f"{name}.weight_scale_2")) module.weight = nn.Parameter(_nvfp4_dequantize_eager( module.weight, scale, global_scale, 16), requires_grad=False) def _load_dflash_lm_head(model: nn.Module, base_model_dir: str, device: str, *, tie_word_embeddings: bool = True) -> None: """Load a DFlash-style draft lm_head from the base model checkpoint. JetSpec reuses the same DFlash draft-head module and official JetSpec draft checkpoints may also omit ``lm_head.*``. Deterministic source selection (matching MTP pattern): 1. Explicit ``lm_head.weight`` from the base checkpoint. 2. Embedding fallback *only* when ``tie_word_embeddings=True``. 3. Otherwise fail loudly — untied models must not use embeddings. This helper is used only for DFlash-style draft checkpoints that do not carry ``lm_head.*`` tensors. Quantized checkpoints, and dense checkpoints that explicitly save ``lm_head.weight``, are loaded by the generic checkpoint loader and must not be overwritten from the base embedding table. """ import logging import os import torch from .models.linear import FP16Linear logger = logging.getLogger(__name__) lm_head = getattr(model, "lm_head", None) if lm_head is None: logger.warning("DFlash-style draft model has no lm_head; skipping.") return from .checkpoint.loader import _build_shard_map shard_map = _build_shard_map(base_model_dir) if not isinstance(lm_head, FP16Linear): _load_dflash_quantized_lm_head(model, lm_head, shard_map, base_model_dir, device) return # --- Determine source key with strict priority --- lm_head_candidates = [ "lm_head.weight", "model.lm_head.weight", "language_model.lm_head.weight", "model.language_model.lm_head.weight", ] embed_candidates = [ "model.embed_tokens.weight", "embed_tokens.weight", "model.language_model.embed_tokens.weight", "language_model.model.embed_tokens.weight", ] # Priority 1: explicit lm_head.weight from base checkpoint source_key = None source_type = None for cand in lm_head_candidates: if cand in shard_map: source_key = cand source_type = "lm_head" break # Priority 2: embedding fallback only if tie_word_embeddings if source_key is None: if tie_word_embeddings: for cand in embed_candidates: if cand in shard_map: source_key = cand source_type = "tied_embedding" break else: raise ValueError( "DFlash lm_head: base model at %s has " "tie_word_embeddings=False but no lm_head.weight found. " "Cannot safely fall back to embed_tokens." % base_model_dir) if source_key is None: raise ValueError( "Cannot find lm_head.weight or embed_tokens.weight in " "base model at %s." % base_model_dir) shard_path = shard_map[source_key] if source_type == "tied_embedding": logger.info("DFlash lm_head source: %s (tied fallback) from %s", source_key, os.path.basename(shard_path)) else: logger.info("DFlash lm_head source: %s from %s", source_key, os.path.basename(shard_path)) source_weight = _read_dflash_checkpoint_tensor(shard_path, source_key, device) # --- Copy weight into model's dense lm_head --- if source_weight.shape != lm_head.weight.shape: raise ValueError( f"DFlash lm_head shape mismatch: source={source_weight.shape} " f"vs lm_head={lm_head.weight.shape}") with torch.no_grad(): lm_head.weight.copy_(source_weight.to(lm_head.weight.dtype)) def _load_dflash_quantized_lm_head(model: nn.Module, lm_head: nn.Module, shard_map: Dict[str, str], base_model_dir: str, device: str) -> None: """Load quantized lm_head sidecar tensors shared from the base checkpoint.""" import logging import os from .checkpoint.loader import _set_tensor logger = logging.getLogger(__name__) source_prefixes = ( "lm_head", "model.lm_head", "language_model.lm_head", "model.language_model.lm_head", ) target_state = lm_head.state_dict() missing = [] source_by_target = {} optional_tensors = {"g_idx", "int4_act_perm", "pre_quant_scale"} for target_name in target_state: source_names = [target_name] if target_name == "weight_scale": source_names.append("weight_scale_inv") source_key = next((f"{prefix}.{source_name}" for prefix in source_prefixes for source_name in source_names if f"{prefix}.{source_name}" in shard_map), None) if source_key is None: if target_name not in optional_tensors: missing.append(target_name) continue source_by_target[target_name] = source_key if missing: raise ValueError( "DFlash quantized lm_head fallback requires base checkpoint tensors " "for %s; missing %s in %s." % (", ".join( target_state.keys()), ", ".join(missing), base_model_dir)) loaded_tensors = [] for target_name, source_key in source_by_target.items(): target_tensor = target_state[target_name] source_tensor = _read_dflash_checkpoint_tensor(shard_map[source_key], source_key, device) source_tensor = _normalize_dflash_lm_head_tensor_shape( target_name, source_tensor, target_tensor) loaded_tensors.append((target_name, source_key, source_tensor)) loaded = [] for target_name, source_key, source_tensor in loaded_tensors: if not _set_tensor(model, f"lm_head.{target_name}", source_tensor, mapping=model.config.mapping): raise ValueError( f"Failed to assign DFlash lm_head tensor {target_name!r}.") loaded.append(source_key) logger.info( "DFlash lm_head source: quantized base checkpoint buffers from %s", ", ".join(sorted({os.path.basename(shard_map[key]) for key in loaded}))) def _read_dflash_checkpoint_tensor(shard_path: str, key: str, device: str): import torch from safetensors import safe_open if shard_path.endswith(".bin"): state = torch.load(shard_path, map_location=device, weights_only=True) return state[key] with safe_open(shard_path, framework="pt", device=device) as f: return f.get_tensor(key) def _normalize_dflash_lm_head_tensor_shape(target_name: str, source, target): if source.shape == target.shape: return source if source.numel() == 1 and target.numel() == 1: return source.reshape(target.shape) raise ValueError( "DFlash quantized lm_head shape mismatch for %s: source=%s vs " "lm_head=%s" % (target_name, source.shape, target.shape)) def _checkpoint_has_dflash_lm_head(model_dir: str, key_remap=None) -> bool: """Return whether a cached-draft checkpoint owns lm_head tensors. ``key_remap`` defaults to the DFlash remap; DSpark passes its own so the check runs against the same key view the loader will use. """ from .checkpoint.loader import _build_shard_map remap = key_remap if key_remap is not None else _dflash_key_remap for key in _build_shard_map(model_dir): mapped = remap(key) if mapped is not None and mapped.startswith("lm_head."): return True return False def _checkpoint_has_draft_vocab_map(model_dir: str) -> bool: """Return whether a draft checkpoint ships a ``d2t`` reduced-vocabulary map.""" from .checkpoint.loader import _build_shard_map return "d2t" in _build_shard_map(model_dir) def _dflash_key_remap(key: str) -> "str | None": """Remap DFlash draft checkpoint keys.""" if "rotary_emb" in key: return None if key == "d2t": return "draft_vocab_map" if key == "t2d": return None return key _LILICORR_SENTINEL_DTYPES = (torch.float16, torch.bfloat16, torch.float32) def _require_unscaled_lilicorr_embedding(base_config) -> None: """Reject a base whose runtime embedding table carries a folded scale. The LiLiCorr draft gathers raw rows of that table for its candidates. """ from .checkpoint.checkpoint_utils import runtime_embedding_scale scale = runtime_embedding_scale(base_config) if scale != 1.0: raise ValueError( "LiLiCorr drafts gather raw rows of the runtime embedding table, " f"but this base pre-scales it by {scale}") def _lilicorr_sentinel_tensors(model: nn.Module): """Yield floating tensors that the draft or base checkpoint must fill.""" for name, tensor in itertools.chain(model.named_parameters(), model.named_buffers()): if (tensor.dtype in _LILICORR_SENTINEL_DTYPES and name.rsplit( ".", 1)[-1] not in _OPTIONAL_SCALE_DEFAULT_BUFFERS): yield name, tensor def _fill_lilicorr_load_sentinels(model: nn.Module) -> None: """NaN-fill checkpoint-backed floating tensors so a missing tensor cannot pass as data.""" with torch.no_grad(): for _, tensor in _lilicorr_sentinel_tensors(model): if tensor.device.type != "meta": tensor.fill_(float("nan")) def _assert_lilicorr_weights_loaded(model: nn.Module) -> None: """Raise ``ValueError`` naming every LiLiCorr draft tensor the load left unfilled. Unfilled means still holding the NaN load sentinel, still on the meta device, or an FP16 linear whose weight is still packed NVFP4. """ from .models.linear import FP16Linear unfilled = [ name for name, tensor in _lilicorr_sentinel_tensors(model) if tensor.device.type == "meta" or bool(torch.isnan(tensor).any()) ] unfilled += [ f"{name}.weight" for name, module in model.named_modules() if isinstance(module, FP16Linear) and not module.weight.is_floating_point() ] if unfilled: raise ValueError( "LiLiCorr draft tensors were not loaded from the checkpoint: " + ", ".join(unfilled)) def _check_lilicorr_coverage(model: nn.Module, shard_map: Dict[str, str], key_remap) -> None: """Raise unless every draft tensor is consumed and every packed model tensor is provided. The generic loader skips unknown keys and leaves absent tensors at their initial values. NaN sentinels cover the floating tensors; this covers stray checkpoint keys and the packed NVFP4 weights and block scales. """ targets = dict( itertools.chain(model.named_parameters(), model.named_buffers())) provided = set() unconsumed = [] consumed = 0 for key in shard_map: mapped = key_remap(key) if mapped is None: continue if mapped not in targets: unconsumed.append(key) continue provided.add(mapped) consumed += 1 missing = [ name for name, tensor in targets.items() if tensor.dtype not in _LILICORR_SENTINEL_DTYPES and name not in provided and not name.startswith("lm_head.") and name != "draft_vocab_map" ] if unconsumed or missing: raise ValueError( "LiLiCorr draft checkpoint does not match the model: unconsumed " f"checkpoint tensors [{', '.join(unconsumed)}]; missing packed " f"tensors [{', '.join(missing)}]") logging.getLogger(__name__).info( "LiLiCorr: consumed %d of %d draft checkpoint tensors", consumed, len(shard_map)) def _finish_lilicorr_load(model: nn.Module, draft_dir: str, key_remap) -> None: """Verify that every draft tensor loaded and build the lattice attention bias.""" from .checkpoint.loader import _build_shard_map _check_lilicorr_coverage(model, _build_shard_map(draft_dir), key_remap) model.lilicorr.materialize() _assert_lilicorr_weights_loaded(model) def _muse_glimmer_dflash_key_remap(key: str) -> "str | None": """Map the Muse-Glimmer assistant fusion projector onto DFlash names.""" if key.startswith("encoder.fc."): return key.removeprefix("encoder.") if key.startswith("encoder.output_norm_enc."): return "hidden_norm." + key.removeprefix("encoder.output_norm_enc.") return _dflash_key_remap(key) def _domino_key_remap(key: str) -> "str | None": """Remap Domino draft checkpoint keys for the backbone engine. The GRU tensors ship as a runtime sidecar and embed_proj is split by ``_load_domino_pool_weights``, so the generic loader skips them. """ if "rotary_emb" in key: return None if key.startswith(("prefix_gru.", "embed_proj.")): return None return key def _load_domino_pool_weights(model: nn.Module, draft_dir: str, device: str) -> None: """Fill the in-engine Domino pool weights from the draft checkpoint. The checkpoint stores embed_proj.0 as one [proj_dim, H + gru_hidden] matrix; the engine consumes only the draft-hidden half (the state half ships in the runtime sidecar), and embed_proj.2 becomes the codebook. """ import torch from .checkpoint.loader import _build_shard_map config = model.config shard_map = _build_shard_map(draft_dir) for key in ("embed_proj.0.weight", "embed_proj.2.weight"): if key not in shard_map: raise ValueError( f"Domino draft checkpoint at {draft_dir} is missing {key}.") w0 = _read_dflash_checkpoint_tensor(shard_map["embed_proj.0.weight"], "embed_proj.0.weight", device) expected_w0 = (config.domino_proj_dim, config.hidden_size + config.domino_gru_hidden_dim) if tuple(w0.shape) != expected_w0: raise ValueError( f"Domino embed_proj.0.weight must have shape {expected_w0}, " f"got {tuple(w0.shape)}") codebook = _read_dflash_checkpoint_tensor(shard_map["embed_proj.2.weight"], "embed_proj.2.weight", device) expected_codebook = (config.vocab_size, config.domino_proj_dim) if tuple(codebook.shape) != expected_codebook: raise ValueError( f"Domino embed_proj.2.weight must have shape {expected_codebook}, " f"got {tuple(codebook.shape)}") pool = model.domino_pool with torch.no_grad(): pool.z_projection.weight.copy_(w0[:, :config.hidden_size].to( pool.z_projection.weight.dtype)) pool.codebook.copy_(codebook.to(pool.codebook.dtype)) def _dspark_key_remap(key: str) -> "str | None": """Remap DSpark draft checkpoint keys for the backbone engine. Markov and confidence tensors are exported as sidecars rather than baked into the backbone engine, so the generic loader may skip them. """ if "rotary_emb" in key: return None if key == "d2t": return "draft_vocab_map" if key == "t2d": return None return key def _mtp_key_remap(key: str, *, tie_word_embeddings: bool, share_base_lm_head: bool = False) -> "str | None": """Remap MTP checkpoint keys for the draft model. ``share_base_lm_head`` sources the draft head from the base model (``lm_head.*`` tensors, or the embedding table when tied); enable it only when the checkpoint has no ``mtp.lm_head.*`` of its own. """ if key.startswith("mtp."): return key[len("mtp."):] if share_base_lm_head and key.startswith("lm_head."): return key if share_base_lm_head and tie_word_embeddings and key in ( "model.embed_tokens.weight", "model.language_model.embed_tokens.weight"): return "lm_head.weight" return None