# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Model configuration parser for quantized LLM checkpoints.
Architecture fields are loaded with HuggingFace ``AutoConfig`` via
:func:`checkpoint_utils.load_checkpoint_config_dicts` (root + promoted LLM
dict; VL nested promotion applies to the LLM dict). Quantization is merged from
``hf_quant_config.json`` or
embedded ``quantization_config`` via helpers in this module.
All model-specific feature flags (``attention_bias``, ``head_dim``, etc.)
are read from the resulting config dict. The ``has_qk_norm`` flag is
auto-detected from the presence of ``q_norm`` weight keys in the checkpoint
index; no model-type string comparisons are used.
Supported quantization formats
-------------------------------
fp16 - plain bfloat16/float16 weights (no quantization)
fp8 - FP8 E4M3 per-tensor static quantization
nvfp4 - NVFP4 per-group quantization with FP8 group scales
nvfp4_a16 - weight-only NVFP4 (W4A16 / ModelOpt ``W4A16_NVFP4``)
int4_awq - AWQ INT4 group quantization (column-packed int32 checkpoints)
int4_awq_modelopt - W4A16_AWQ pre-packed uint8 ``[out//2, in]`` checkpoints
int4_gptq - GPTQ INT4 group quantization
int8_gptq - GPTQ/compressed-tensors INT8 weight-only quantization
int8_sq - INT8 SmoothQuant W8A8 per-channel
mixed_precision - per-layer mixed quantization (from ``hf_quant_config``)
Hybrid model support
--------------------
When ``config.json`` contains a ``layers_block_type`` list (e.g.
``["attention", "attention", "mamba", ...]``), a ``MambaConfig`` is parsed
and ``ModelConfig.layer_types`` reflects the per-layer block type. Mamba
parameters (``mamba_num_heads``, ``mamba_head_dim``, ``ssm_state_size``,
``conv_dim``, ``conv_kernel``) are read from the config. When ``conv_dim``
is absent (e.g. NemotronH-4B-BF16), it is derived from the ``conv1d.weight``
shape in the checkpoint to break the circular dependency with ``n_groups``.
"""
import fnmatch
import json
import math
import os
from dataclasses import asdict, dataclass, field, replace
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
if TYPE_CHECKING:
import torch
from .checkpoint.checkpoint_utils import load_checkpoint_config_dicts
from .checkpoint.component_prefixes import VISUAL_PREFIXES
from .dflash import DFlashVersion, resolve_dflash_contract
# ---------------------------------------------------------------------------
# Quantization type constants
# ---------------------------------------------------------------------------
QUANT_FP16 = "fp16"
QUANT_FP8 = "fp8"
QUANT_MXFP8 = "mxfp8"
QUANT_NVFP4 = "nvfp4"
# Weight-only NVFP4 (W4A16): ModelOpt ``W4A16_NVFP4`` — 4-bit float weights,
# FP16 activations. Distinct from QUANT_NVFP4 (W4A4); routed to the dense/MoE
# Marlin FP16xE2M1 kernels.
QUANT_NVFP4_A16 = "nvfp4_a16"
QUANT_INT4_AWQ = "int4_awq"
QUANT_INT4_AWQ_MODELOPT = "int4_awq_modelopt"
QUANT_INT4_GPTQ = "int4_gptq"
QUANT_INT8_GPTQ = "int8_gptq"
QUANT_INT8_SQ = "int8_sq"
QUANT_Q8_0 = "q8_0"
QUANT_IQ2_XXS = "iq2_xxs"
# ``quant_algo`` value in hf_quant_config for MIXED_PRECISION only.
# After parsing, :attr:`QuantConfig.quant_type` is always a concrete type
# (dominant algo); per-layer differences live in ``layer_overrides``.
QUANT_MIXED = "mixed_precision"
# Default RoPE base frequency (used when config omits rope_theta)
_DEFAULT_ROPE_THETA = 10000.0
# Model families whose per-head QK RMSNorm runs AFTER RoPE (HunYuan V1);
# the default (Qwen3 convention) normalizes before rotation.
_QK_NORM_POST_ROPE_MODEL_TYPES = frozenset({"hunyuan_v1_dense"})
# Model families that store the per-head QK norms under the HunYuan
# ``query_layernorm`` / ``key_layernorm`` key names. Kept in sync with the
# ``_hunyuan_key_remap`` dispatch in model.py — QK-norm detection must not
# fire for families whose loader would not remap these keys onto q/k_norm.
_QUERY_LAYERNORM_KEY_MODEL_TYPES = frozenset({"hunyuan_v1_dense"})
# Layer-type labels
LAYER_ATTN = "attention"
LAYER_MAMBA = "mamba"
LAYER_MLP = "mlp"
LAYER_GDN = "gdn" # GatedDeltaNet linear attention (Qwen3.5)
LAYER_MOE = "moe"
_VALID_ATTENTION_LAYER_TYPES = ("sliding_attention", "full_attention")
# NemotronH ``hybrid_override_pattern`` / ``mtp_hybrid_override_pattern`` chars.
_HYBRID_PATTERN_MAP = {
"M": LAYER_MAMBA,
"-": LAYER_MLP,
"*": LAYER_ATTN,
"E": LAYER_MOE,
}
_DIFFUSION_GEMMA_MODEL_TYPES = frozenset({
"diffusion_gemma",
"diffusion_gemma_text",
"diffusiongemma",
})
_DEFAULT_DIFFUSION_MAX_DENOISING_STEPS = 48
def _is_diffusion_gemma_model_type(model_type: str) -> bool:
return str(model_type).lower() in _DIFFUSION_GEMMA_MODEL_TYPES
def _is_diffusion_gemma_config(root_dict: Dict[str, Any],
llm_dict: Dict[str, Any]) -> bool:
model_type = str(
root_dict.get("model_type", llm_dict.get("model_type", ""))).lower()
architectures_value = (root_dict.get("architectures")
or llm_dict.get("architectures") or [])
if isinstance(architectures_value, str):
architectures = [architectures_value]
else:
architectures = [str(x) for x in architectures_value]
return (_is_diffusion_gemma_model_type(model_type)
or any("DiffusionGemma" in arch for arch in architectures))
def _is_gemma4_model_type(model_type: str) -> bool:
model_type = str(model_type)
return (model_type.startswith("gemma4")
or _is_diffusion_gemma_model_type(model_type))
def _is_gemma4_assistant_model_type(model_type: str) -> bool:
return str(model_type) in ("gemma4_assistant", "gemma4_unified_assistant")
def _is_muse_glimmer_model_type(model_type: str) -> bool:
"""Return whether a model type belongs to Muse-Glimmer."""
return str(model_type) in ("muse_glimmer", "muse_glimmer_text")
def _check_num_attention_heads(num_attn_heads: int) -> None:
if num_attn_heads <= 0:
raise ValueError("num_attention_heads must be a positive integer, "
f"got {num_attn_heads!r}")
def _get_rope_theta(llm_dict: Dict[str, Any]) -> float:
"""Extract rope_theta from config dict.
Some models (e.g. Qwen3) store rope_theta inside ``rope_scaling`` or
``rope_parameters`` rather than as a top-level key, so fall back to those
nested dicts before returning the default 10 000.
"""
if llm_dict.get("rope_theta") is not None:
return float(llm_dict["rope_theta"])
for key in ("rope_scaling", "rope_parameters"):
nested = llm_dict.get(key)
if not isinstance(nested, dict):
continue
if nested.get("rope_theta") is not None:
return float(nested["rope_theta"])
for attention_type in ("full_attention", "sliding_attention"):
attention_params = nested.get(attention_type)
if (isinstance(attention_params, dict)
and attention_params.get("rope_theta") is not None):
return float(attention_params["rope_theta"])
return _DEFAULT_ROPE_THETA
def _normalize_rope_scaling_for_config(
rope_params: Optional[Dict[str, Any]]) -> Optional[dict]:
"""Normalize one RoPE parameter block for runtime config export."""
if not isinstance(rope_params, dict):
return None
normalized = dict(rope_params)
rope_type = normalized.get("rope_type", normalized.get("type"))
if normalized.get("mrope_section") is not None:
normalized["rope_type"] = "mrope"
normalized["type"] = "mrope"
elif rope_type is not None:
normalized.setdefault("rope_type", rope_type)
normalized.setdefault("type", rope_type)
return normalized
def _select_rope_scaling(llm_dict: Dict[str, Any]) -> Optional[dict]:
"""Select the single-RoPE fallback block from raw config metadata."""
for key in ("rope_scaling", "rope_parameters"):
nested = llm_dict.get(key)
if not isinstance(nested, dict):
continue
if isinstance(nested.get("full_attention"), dict):
return _normalize_rope_scaling_for_config(nested["full_attention"])
if isinstance(nested.get("sliding_attention"), dict):
return _normalize_rope_scaling_for_config(
nested["sliding_attention"])
return _normalize_rope_scaling_for_config(nested)
return None
def _runtime_rope_config_from_params(llm_dict: Dict[str, Any],
rope_params: Dict[str, Any]) -> dict:
"""Build one runtime RoPE config block from raw checkpoint metadata."""
out = {
"rope_theta":
float(
rope_params.get("rope_theta",
llm_dict.get("rope_theta", _DEFAULT_ROPE_THETA))),
"rope_scaling":
_normalize_rope_scaling_for_config(rope_params),
"partial_rotary_factor":
float(
rope_params.get("partial_rotary_factor",
llm_dict.get("partial_rotary_factor", 1.0))),
"max_position_embeddings":
int(llm_dict.get("max_position_embeddings", 4096)),
}
return out
def _get_dual_rope_configs(llm_dict: Dict[str, Any]) -> dict[str, dict]:
"""Return explicit sliding/full runtime RoPE configs when present."""
rope_parameters = llm_dict.get("rope_parameters")
layer_types = {
str(layer_type)
for layer_type in llm_dict.get("layer_types", [])
}
if not isinstance(rope_parameters, dict):
return {}
if not {"sliding_attention", "full_attention"} <= layer_types:
return {}
sliding_params = rope_parameters.get("sliding_attention")
full_params = rope_parameters.get("full_attention")
if not isinstance(sliding_params, dict) or not isinstance(
full_params, dict):
return {}
return {
"sliding_rope_config":
_runtime_rope_config_from_params(llm_dict, sliding_params),
"full_rope_config":
_runtime_rope_config_from_params(llm_dict, full_params),
}
def _parse_attention_layer_types(config: dict, num_hidden_layers: int,
model_type: str) -> List[str]:
"""Preserve per-layer sliding/full attention labels when required."""
raw = config.get("layer_types")
if not (_is_gemma4_model_type(model_type)
or _is_muse_glimmer_model_type(model_type)):
return []
if not isinstance(raw, list):
raise ValueError(
f"{model_type} requires layer_types with one sliding/full "
"attention entry per layer.")
if len(raw) != num_hidden_layers:
raise ValueError(
f"{model_type} layer_types length must match num_hidden_layers: "
f"{len(raw)} vs {num_hidden_layers}.")
attention_layer_types: List[str] = []
for layer_idx, layer_type in enumerate(raw):
layer_type = str(layer_type)
if layer_type not in _VALID_ATTENTION_LAYER_TYPES:
raise ValueError(
f"{model_type} layer_types entries must be sliding_attention "
"or full_attention; "
f"got {layer_type!r} at layer {layer_idx}.")
attention_layer_types.append(layer_type)
return attention_layer_types
def _get_attention_scaling(llm_dict: Dict[str, Any], head_dim: int,
default_val: float) -> float:
"""Resolve the absolute multiplier applied to QK^T before softmax.
Args:
llm_dict: Checkpoint configuration containing an optional supported
attention-scale alias.
head_dim: Per-head query/key dimension.
default_val: Required model-family fallback.
Returns:
The finite, positive attention scale supplied by the checkpoint, or
the caller-selected fallback.
Raises:
ValueError: If an explicit checkpoint scale is not finite and
positive, or if ``head_dim`` is not positive.
"""
if head_dim <= 0:
raise ValueError(f"head_dim must be positive; got {head_dim}")
for key in ("attention_scaling", "qk_scale", "scaling"):
if llm_dict.get(key) is not None:
attention_scale = float(llm_dict[key])
if not math.isfinite(attention_scale) or attention_scale <= 0.0:
raise ValueError(
f"{key} must be finite and positive; got {llm_dict[key]!r}"
)
return attention_scale
attention_scale = float(default_val)
if not math.isfinite(attention_scale) or attention_scale <= 0.0:
raise ValueError(
f"default_val must be finite and positive; got {default_val!r}")
return attention_scale
def _get_rms_norm_eps(llm_dict: Dict[str, Any], model_type: str) -> float:
if str(model_type).lower().startswith("nemotron_h"):
return llm_dict.get(
"rms_norm_eps",
llm_dict.get("norm_eps", llm_dict.get("layer_norm_epsilon", 1e-6)))
return llm_dict.get("rms_norm_eps", 1e-6)
def _get_embedding_scale(llm_dict: Dict[str, Any], model_type: str,
hidden_size: int) -> float:
"""Return the scale folded into runtime token embeddings."""
for key in ("embedding_scale", "embed_scale", "scalar_embed_scale"):
if llm_dict.get(key) is not None:
return float(llm_dict[key])
if _is_gemma4_model_type(model_type):
return math.sqrt(float(hidden_size))
return 1.0
def _get_has_value_norm(llm_dict: Dict[str, Any], model_type: str) -> bool:
"""Return whether attention values use Gemma-style RMSNorm."""
for key in ("has_value_norm", "has_v_norm", "value_norm"):
if llm_dict.get(key) is not None:
return bool(llm_dict[key])
return _is_gemma4_model_type(model_type)
@dataclass
class Mapping:
"""Tensor-parallel placement for one exported or loaded rank.
``tp_size`` / ``tp_rank`` are the only supported model-sharding
coordinates in the current Edge-LLM multi-device flow. Future CP/EP/PP/DP
work should add its own mapping fields together with matching export,
runtime, and validation support.
"""
world_size: int = 1
rank: int = 0
tp_size: int = 1
tp_rank: int = 0
@dataclass
class ActionConfig:
"""Action expert hyper-parameters for Alpamayo models.
Used only for the action expert ONNX export and its sidecar config.json.
These fields do NOT feed into the LLM config.json.
"""
rope_theta: float = 5_000_000.0
mrope_section: List[int] = field(default_factory=lambda: [24, 20, 20])
mrope_interleaved: bool = True
num_hidden_layers: int = 0
num_attention_heads: int = 0
num_key_value_heads: int = 0
head_dim: int = 128
attention_scaling: Optional[float] = None
hidden_size: int = 0
intermediate_size: int = 0
rms_norm_eps: float = 1e-6
num_traj_tokens: int = 1000
traj_token_start: int = 0
n_diffusion_tokens: int = 64
in_proj_hidden_size: int = 512
in_proj_num_enc_layers: int = 2
in_proj_max_freq: float = 100.0
in_proj_num_fourier_feats: int = 20
kv_cache_scales: Optional[List[List[float]]] = None
def __post_init__(self) -> None:
"""Resolve and validate attention scaling for direct construction.
Raises:
ValueError: If the head dimension or explicit scale is invalid.
"""
scale_config = ({} if self.attention_scaling is None else {
"attention_scaling": self.attention_scaling
})
self.attention_scaling = _get_attention_scaling(
scale_config, self.head_dim, 1.0 / (float(self.head_dim)**0.5))
_DEFAULT_QUANTIZE_ACTIVATIONS = True
def set_default_quantize_activations(value: bool) -> None:
"""Set :attr:`QuantConfig.quantize_activations` for configs parsed later.
The export CLI calls this once from its argument parsing. A whole export can
build several QuantConfigs (backbone plus any draft model), all of them
parsed from their checkpoints afterwards, so one assignment covers the run
without threading the flag through every ``_export_*`` entry point.
"""
global _DEFAULT_QUANTIZE_ACTIVATIONS
_DEFAULT_QUANTIZE_ACTIVATIONS = value
[docs]
@dataclass
class QuantConfig:
"""Quantization parameters extracted from the checkpoint config."""
quant_type: str = QUANT_FP16
# group_size: 1 = per-tensor/per-channel, 16 for NVFP4, 128 for AWQ
group_size: int = 1
# GPTQ checkpoints are not consistent about whether qzeros stores the
# actual zero point or (zero point - 1). The loader uses:
# actual_zero = stored_zero + gptq_zero_point_offset.
gptq_zero_point_offset: int = 1
# kv_cache_quant: "fp8" when KV-cache is quantised, None otherwise
kv_cache_quant: Optional[str] = None
# visual_mha_quant: "fp8" when the ViT visual attention (Q*K^T / P*V
# matmuls) is quantised, None otherwise. Orthogonal to kv_cache_quant
# (which only gates the LLM KV cache).
visual_mha_quant: Optional[str] = None
# module names excluded from quantisation (typically ["lm_head"])
excluded: List[str] = field(default_factory=list)
# Per-layer quant type overrides for MIXED_PRECISION checkpoints.
# Maps a module name (e.g. "lm_head") to a quant-type string.
# make_linear() uses module_name together with ``excluded`` and (for
# lm_head) ``ModelConfig.tie_word_embeddings`` to pick FP16 vs overrides.
layer_overrides: dict = field(default_factory=dict)
# Per-layer group sizes use the same normalized names as layer_overrides.
layer_group_sizes: dict = field(default_factory=dict)
# True when quant_algo is MIXED_PRECISION: unlisted modules are FP16.
is_mixed_precision: bool = False
# False selects the NVFP4-A16 kernel for packed NVFP4 weights. Other
# activation-quantized formats omit their activation Q-DQ pair for accuracy
# debugging. The default reproduces the checkpoint's W4A4/W8A8 recipe.
quantize_activations: bool = field(
default_factory=lambda: _DEFAULT_QUANTIZE_ACTIVATIONS)
@property
def is_quantized(self) -> bool:
return self.quant_type != QUANT_FP16
@property
def uses_nvfp4_weights(self) -> bool:
"""True if any linear uses NVFP4 weights (dominant quant or layer override)."""
nvfp4_types = (QUANT_NVFP4, QUANT_NVFP4_A16)
if self.quant_type in nvfp4_types:
return True
return any(v in nvfp4_types for v in self.layer_overrides.values())
@property
def uses_mxfp8_weights(self) -> bool:
"""True if any linear uses MXFP8 weights (dominant quant or layer override)."""
if self.quant_type == QUANT_MXFP8:
return True
return any(v == QUANT_MXFP8 for v in self.layer_overrides.values())
def module_quant_type(module_name: str, model_config: "ModelConfig") -> str:
"""Return the effective quant type ``make_linear`` will pick for *module_name*.
Single source of truth for "what precision does this module's Linear end
up at?" — used by :func:`make_linear` to pick the Linear class, and by
the ONNX exporter to validate that LM-head externalization is only
requested for an fp16 head. Lookup matches the names used elsewhere:
``ModelConfig.quant.excluded`` and ``layer_overrides`` keys are already
normalized (VL prefixes / ``model.`` stripped) when parsed, and callers
pass the same short ``module_name`` that ``make_linear`` receives
(e.g. ``"lm_head"``).
"""
quant = model_config.quant
if module_name and any(
fnmatch.fnmatchcase(module_name, pattern)
for pattern in quant.excluded):
return QUANT_FP16
# Tied lm_head with no explicit override and an unquantized backbone has
# no separate lm_head.weight in the checkpoint; treat it as fp16 so the
# weight can be cloned from embed_tokens after loading.
if (module_name == "lm_head" and model_config.tie_word_embeddings
and "lm_head" not in quant.layer_overrides
and quant.quant_type == QUANT_FP16):
return QUANT_FP16
quant_type = quant.quant_type
if module_name and quant.layer_overrides:
fallback = QUANT_FP16 if quant.is_mixed_precision else quant_type
quant_type = quant.layer_overrides.get(module_name, fallback)
return quant_type
def module_quant_group_size(module_name: str,
model_config: "ModelConfig") -> int:
"""Return the checkpoint group size for a quantized linear module."""
quant = model_config.quant
return int(quant.layer_group_sizes.get(module_name, quant.group_size))
@dataclass
class MambaConfig:
"""Mamba-layer hyper-parameters for hybrid models."""
num_heads: int # mamba_num_heads
head_dim: int # mamba_head_dim
ssm_state_size: int # ssm_state_size
conv_dim: int # conv_dim (total convolution channel count)
conv_kernel: int # conv1d kernel size (default 4)
n_groups: int # number of SSM groups (derived if not in config)
@property
def intermediate_size(self) -> int:
"""num_heads x head_dim - the Mamba intermediate feature dimension."""
return self.num_heads * self.head_dim
@dataclass
class GdnConfig:
"""GatedDeltaNet (GDN) hyper-parameters for Qwen3.5 hybrid models.
GDN layers use a gated delta-net linear attention mechanism with
fused QKV projection through causal conv1d.
"""
num_key_heads: int # linear_num_key_heads
num_value_heads: int # linear_num_value_heads
key_head_dim: int # linear_key_head_dim
value_head_dim: int # linear_value_head_dim
conv_kernel: int # linear_conv_kernel_dim (default 4)
output_gate_type: str # output_gate_type or hidden_act (ACT2FN name)
@property
def key_dim(self) -> int:
"""Total key dimension (num_key_heads * key_head_dim)."""
return self.num_key_heads * self.key_head_dim
@property
def value_dim(self) -> int:
"""Total value dimension (num_value_heads * value_head_dim)."""
return self.num_value_heads * self.value_head_dim
@property
def conv_dim(self) -> int:
"""Total conv1d channel count: key + key + value (QKV fused)."""
return self.key_dim + self.key_dim + self.value_dim
@dataclass
class DiffusionConfig:
"""Block-diffusion generation parameters parsed at export time."""
diffusion_family: str = "uniform_renoise"
canvas_length: int = 256
max_denoising_steps: int = _DEFAULT_DIFFUSION_MAX_DENOISING_STEPS
t_max: float = 0.8
t_min: float = 0.4
sampler_type: str = "entropy_bound"
entropy_bound: float = 0.1
entropy_threshold: float = 0.005
stability_window: int = 2
self_conditioning_enabled: bool = True
self_conditioning_repr: str = "embeds"
supported_modalities: List[str] = field(default_factory=lambda: ["text"])
@classmethod
def from_hf(
cls,
hf_config: Dict[str, Any],
gen_config: Optional[Dict[str, Any]] = None) -> "DiffusionConfig":
gen_config = gen_config or {}
sampler_cfg = gen_config.get("sampler_config", {}) or {}
sampler_name = str(sampler_cfg.get("_cls_name", "EntropyBound"))
sampler_type = ("entropy_bound"
if "entropybound" in sampler_name.lower().replace(
"_", "") else sampler_name.lower())
return cls(
canvas_length=int(
hf_config.get("canvas_length",
gen_config.get("canvas_length", 256))),
max_denoising_steps=int(
gen_config.get("max_denoising_steps",
_DEFAULT_DIFFUSION_MAX_DENOISING_STEPS)),
t_max=float(gen_config.get("t_max", 0.8)),
t_min=float(gen_config.get("t_min", 0.4)),
sampler_type=sampler_type,
entropy_bound=float(
sampler_cfg.get("entropy_bound",
gen_config.get("entropy_bound", 0.1))),
entropy_threshold=float(
gen_config.get("entropy_threshold",
gen_config.get("confidence_threshold", 0.005))),
stability_window=int(
gen_config.get("stability_window",
gen_config.get("stability_threshold", 2))),
)
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
[docs]
@dataclass
class ModelConfig:
"""Flat model hyper-parameter config consumed by module builders."""
# ------------------------------------------------------------------ arch
model_type: str # HF architecture name, e.g. "qwen3", "llama"
hidden_size: int
num_hidden_layers: int
num_attention_heads: int
num_key_value_heads: int
intermediate_size: int
head_dim: int
rms_norm_eps: float
vocab_size: int
rope_theta: float
max_position_embeddings: int
default_attention_scale: float
# RoPE scaling config (e.g. {"type": "dynamic", "factor": 2.0} for Qwen2).
# None means no scaling (standard RoPE).
rope_scaling: Optional[dict] = None
# For longrope: original context window size before extension.
original_max_position_embeddings: Optional[int] = None
# Fraction of head_dim used for RoPE (e.g. 0.75 for phi3/phi4, 1.0 for most others).
partial_rotary_factor: float = 1.0
# Gemma4: head_dim for global (full_attention) layers (0 = same as head_dim)
global_head_dim: int = 0
# Gemma4 E4B: num_key_value_heads for global (full_attention) layers (0 = same as num_key_value_heads)
num_global_key_value_heads: int = 0
# QSA (Qwen Sparse Attention): indexer head dim; QSA layers widen their paged KV pool
# rows by this many columns to persist indexer state (0 = no QSA layers).
indexer_head_dim: int = 0
# Hidden activation name used by architecture-specific auxiliary modules.
hidden_activation: str = "silu"
# CodePredictor: RVQ code groups (lm_heads count = num_code_groups - 1).
num_code_groups: int = 0
# Optional explicit RoPE configs for mixed sliding/full attention stacks.
sliding_rope_config: Optional[dict] = None
full_rope_config: Optional[dict] = None
# ------------------------------------------ model-family feature flags
# Per-head RMSNorm after Q and K projections.
# Auto-detected from checkpoint key names; not inferred from model_type.
has_qk_norm: bool = False
# QK-norm order relative to RoPE. False (Qwen3 convention): norm then
# rotate. True (HunYuan V1): rotate then norm — gamma placement differs,
# so the attention plugin must apply the norm after rotation.
qk_norm_post_rope: bool = False
# Per-head RMSNorm after V projection. Gemma4 stores this norm without
# learned weights, so it is selected from config metadata instead of
# checkpoint key names.
has_value_norm: bool = False
# Bias on q/k/v projections. Read from config.json "attention_bias".
attention_bias: bool = False
# Explicit multiplicative scale applied to QK^T before softmax.
attention_scaling: Optional[float] = None
# Gemma4 full/global attention can use a different per-head dimension
# from sliding attention.
global_head_dim: Optional[int] = None
# Gemma4 K=V full/global attention can use a different KV head count from
# sliding attention.
num_global_key_value_heads: Optional[int] = None
# Gemma4 full/global attention reuses k_proj(hidden_states) as the value
# projection source when enabled.
attention_k_eq_v: bool = False
# Per-Q-head learned attention sink: extra logit merged into softmax denominator.
attention_sink_bias: bool = False
# DiffusionGemma uses one shared backbone with phase-dependent layer scalars.
encoder_layer_scalars: List[float] = field(default_factory=list)
decoder_layer_scalars: List[float] = field(default_factory=list)
self_conditioning_size: int = 0
diffusion: Optional[DiffusionConfig] = None
# Multiplicative scale applied by the HF embedding module.
embedding_scale: float = 1.0
# Final logit softcapping: tanh(logits/cap)*cap. None = disabled.
final_logit_softcapping: Optional[float] = None
# Muse-Glimmer: pre-tanh logit multiplier (applied before the softcap).
output_multiplier: float = 1.0
# Muse-Glimmer: eps for the post-attention / post-FFN sandwich norms
# (None -> fall back to rms_norm_eps).
post_norm_eps: Optional[float] = None
# Weight dtype in the checkpoint
torch_dtype: str = "bfloat16"
# When True, embed_tokens and lm_head share the same weight tensor
tie_word_embeddings: bool = False
# Sliding window attention size; -1 means no sliding window.
sliding_window_size: int = -1
# Which skip-softmax (BLASST) scale tables the engine deploys; each wires
# the matching plugin input (skip_softmax_prefill_scale /
# skip_softmax_decode_scale). Set by the exporter from the resolved
# deployment (tensorrt_edgellm.skip_softmax), never read from the checkpoint.
skip_softmax_prefill_enabled: bool = False
skip_softmax_decode_enabled: bool = False
# Gemma4 Unified 12B+: image placeholder runs use block-causal
# attention during prefill (bidirectional inside each contiguous vision
# run, causal everywhere else). Audio placeholders remain causal.
use_vision_bidirectional_attention: bool = False
# ------------------------------------------ per-layer block types
# One entry per hidden layer: LAYER_ATTN, LAYER_MAMBA, LAYER_MLP, or LAYER_MOE.
layer_types: List[str] = field(default_factory=list)
# Nemotron-H Puzzle: one block definition per layer; MoE widths and
# expert counts are not uniform across the decoder.
block_configs: List[dict] = field(default_factory=list)
# Original attention labels for attention layers: full_attention or sliding_attention.
attention_layer_types: List[str] = field(default_factory=list)
# ------------------------------------------ multimodal deepstack (VL)
# Number of deepstack visual embedding tensors injected into the first N
# hidden layers. Prefer ``vision_config.deepstack_visual_indexes`` length on
# the root config when present; else ``num_deepstack_features`` or fallback.
num_deepstack_features: int = 0
# ----------------------------------------- Qwen3-Omni emitted-tensor layer
# Read by the Qwen3-Omni-specific ``Transformer`` subclasses (dense and
# MoE) to decide which tensor to expose via ``emitted_hidden_states``
# (consumed by their ``CausalLM`` wrappers when
# ``emit_hidden_states = True``):
#
# * ``accept_hidden_layer >= 1`` (and ≤ num_hidden_layers): pre-norm
# output of decoder layer ``accept_hidden_layer - 1``. Matches HF's
# ``outputs.hidden_states[k]`` convention where ``k`` denotes "after
# ``k`` decoder layers" (k=0 is inputs_embeds, k=N is the last layer
# pre-norm). Used by Qwen3-Omni Thinker → Talker: Talker
# consumes ``thinker.hidden_states[accept_hidden_layer]``.
#
# * Default ``-1``: post-final-norm output (= ``model.norm(last_layer)``).
# Used by Qwen3-Omni Talker → CodePredictor: HF reads
# ``hidden_states[0][-1]`` which resolves to the post-norm tensor.
accept_hidden_layer: int = -1
# -------------------------------------------------- quantization config
quant: QuantConfig = field(default_factory=QuantConfig)
# ------------------------------------------ mamba / hybrid config
mamba_cfg: Optional[MambaConfig] = None
# ------------------------------------------ gdn / hybrid config
gdn_cfg: Optional[GdnConfig] = None
# ------------------------------------------ gated attention (Qwen3.5)
attn_output_gate: bool = False
# ------------------------------------------ gated residual (Qwen4-Exp)
# Parallel residual streams ("HyperConnections"); 0 = single residual bus.
hc_count: int = 0
# Bottleneck width of the per-stream read-gate projection.
hc_lowrank: int = 0
# ------------------------------------------ QSA indexer (Qwen4-Exp)
# Block-compressed indexer picking which KV blocks sparse attention reads.
# 0 = dense attention. block_topk = indexer_budget // indexer_compress_ratio.
indexer_n_heads: int = 0
indexer_kv_heads: int = 0
indexer_budget: int = 0
indexer_compress_ratio: int = 0
# ------------------------------------------ PLE (Qwen4-Exp)
# Layers carrying an n-gram embedding injected into the residual, 1-indexed
# as the reference config writes them. Empty for every checkpoint that does
# not use PLE, which is what keeps the whole path optional.
ple_layer_ids: List[int] = field(default_factory=list)
# Width of the gathered n-gram vector the runtime hands to the engine.
ple_embed_dim: int = 0
# Kernel of the depthwise conv that widens the injection; its dilation is
# ``ngram_size``, so its state spans ``(kernel - 1) * ngram_size`` steps.
ple_conv_kernel_size: int = 0
# Longest n-gram order hashed into a row ID, and the hash heads per order.
# The runtime needs both to size the row-ID tensor and the token history.
ngram_size: int = 0
heads_per_ngram: int = 0
# Row blocks the checkpoint stores the table in.
split_ngram_parts: int = 0
# ------------------------------------------ MTP config
mtp_num_hidden_layers: Optional[int] = None
mtp_use_dedicated_embeddings: bool = False
# Nemotron-H MTP: the draft module is a hybrid stack whose layer types come
# from this pattern (e.g. "*E" -> [attention, MoE]); ``mtp_num_hidden_layers``
# is then its length. ``num_nextn_predict_layers`` (the count of MTP prediction
# modules) is folded in during parsing.
mtp_hybrid_override_pattern: Optional[str] = None
# The same draft stack, resolved from either the pattern above or the
# ``mtp_layers_block_type`` list.
mtp_layer_types: List[str] = field(default_factory=list)
# Set by make_mtp_draft_config; the base and its derived draft config are
# otherwise indistinguishable by their fields for dense checkpoints.
is_mtp_draft_flag: bool = False
# When True, the standard CausalLM is exported as the MTP base model variant
# with tree-attention inputs (attention_mask, attention_pos_id) and
# an extra hidden_states output.
mtp_base: bool = False
# ------------------------------------------ Gemma4 MTP config
root_model_type: str = ""
raw_layer_types: List[str] = field(default_factory=list)
rope_parameters: Optional[dict] = None
backbone_hidden_size: int = 0
gemma4_mtp_base: bool = False
gemma4_mtp_draft: bool = False
assistant_hidden_size: int = 0
shares_target_kv: bool = False
has_own_kv_cache: bool = True
constant_draft_positions: bool = False
returns_feedback_hidden: bool = False
use_ordered_embeddings: bool = False
num_centroids: int = 0
centroid_intermediate_top_k: int = 0
sparse_logits_enabled: bool = False
kv_sharing_map: List[dict] = field(default_factory=list)
# ------------------------------------------ EAGLE3 draft config
draft_vocab_size: Optional[int] = None
target_hidden_size: Optional[int] = None
is_eagle3_draft_flag: bool = False
# HF hidden-state indices (input of decoder layer k); Gemma4 bases read
# flat draft IDs as decoder-layer outputs instead.
eagle3_target_layer_ids: List[int] = field(default_factory=list)
eagle3_norm_before_residual: bool = False
eagle3_norm_before_fc: bool = False
# Per-target-layer RMSNorm on the fc input.
eagle_fc_norm: bool = False
# The draft feeds back its final-norm output instead of the pre-norm one.
eagle_norm_output: bool = False
# Low-rank candidate head; 0 means the dense lm_head.
draft_candidate_rank: int = 0
draft_candidate_count: int = 0
# ------------------------------------------ EAGLE3 base config
# When True, the standard CausalLM is exported as an EAGLE3 base model
# with tree-attention inputs (attention_mask, attention_pos_id) and
# an extra hidden_states output (concatenated from 3 selected layers).
eagle_base: bool = False
# ------------------------------------------ DFlash config
# When True, export the standard Qwen3.5 model as the DFlash base with
# tree-attention verify inputs and multi-layer hidden_states output.
dflash_base: bool = False
is_dflash_draft_flag: bool = False
dflash_target_layer_ids: List[int] = field(default_factory=list)
dflash_block_size: int = 16
dflash_mask_token_id: int = 248070
dflash_version: DFlashVersion = DFlashVersion.V1
dflash_overflow_fp32: bool = False
# Run the fc feature projector at the checkpoint's native precision (e.g.
# NVFP4) instead of the default dense-FP16 + FP32 projection. Enabled only
# for targets measured to keep target-hidden well inside FP16 range
# (Nemotron-3.5). Qwen3-8B keeps the FP32 guard (target-hidden ~abs 2e4).
dflash_fc_native_precision: bool = False
# DFlash2 is a distinct linear-path proposal architecture. Its checkpoint
# block size is the runtime default; an engine may profile a larger block.
dflash2_target_layer_ids: List[int] = field(default_factory=list)
dflash2_block_size: int = 8
dflash2_mask_token_id: int = 248070
dflash2_is_causal: bool = False
dflash2_conv_kernel_size: int = 2
dflash2_conv_group_size: int = 16
dflash2_selector_rank: int = 256
dflash2_selector_top_k: int = 16
dflash2_output_multiplier: float = 1.0
dflash2_final_logit_softcapping: float = 0.0
# ------------------------------------------ Domino config
# Domino keeps the DFlash V1 draft ABI and adds a runtime causal
# correction head (GRU over realized-token embeddings + a rank-proj_dim
# vocab codebook). The candidate pool (top-pool_size tokens by base
# logits over the feedback positions) is materialized inside the draft
# engine; the GRU and state projection run as runtime sidecars.
dflash_projector_type: str = ""
domino_gru_hidden_dim: int = 1024
domino_proj_dim: int = 256
domino_prefix_len: int = 1
domino_pool_size: int = 2048
domino_shift_label: bool = False
# LiLiCorr (dflash_projector_type == "lilicorr"): per-slot candidate count.
lilicorr_candidate_topk: int = 0
# LiLiCorr lattice head geometry; logit_scale multiplies cosine scores.
lilicorr_hidden_size: int = 0
lilicorr_num_layers: int = 0
lilicorr_num_heads: int = 0
lilicorr_mlp_ratio: float = 0.0
lilicorr_factor_dim: int = 0
lilicorr_vector_eps: float = 0.0
lilicorr_logit_scale: float = 0.0
# DFlash V1 draft whose block position i predicts token i + 1 (DeepSpec).
dflash_shift_label: bool = False
# ------------------------------------------ JetSpec config
# JetSpec uses the DFlash/DDTree cached-draft contract with causal proposal
# attention inside the draft block. The DFlash-prefixed fields are still
# populated for the shared DFlashDraftModel ONNX implementation.
jetspec_base: bool = False
is_jetspec_draft_flag: bool = False
jetspec_target_layer_ids: List[int] = field(default_factory=list)
jetspec_block_size: int = 16
jetspec_mask_token_id: int = 151669
jetspec_causal_head: bool = False
# ------------------------------------------ DSpark config
# DSpark uses the DFlash-like target-hidden feedback path, then applies
# a sequential Markov/confidence head outside the draft backbone engine.
dspark_base: bool = False
is_dspark_draft_flag: bool = False
dspark_target_layer_ids: List[int] = field(default_factory=list)
dspark_block_size: int = 7
dspark_mask_token_id: int = 151669
dspark_enable_confidence_head: bool = False
dspark_sample_from_anchor: bool = True
dspark_confidence_head_with_markov: bool = False
dspark_markov_head_type: str = ""
dspark_markov_rank: int = 0
dspark_fc_native_precision: bool = False
dspark_causal_proposal: bool = False
# ------------------------------------------ sparse MoE config (Qwen3-style)
# num_experts=0 means dense (no MoE) for Qwen/Mixtral-style keys; Nemotron-H instead reports
# its expert count via n_routed_experts, so n_routed_experts > 0 also indicates MoE.
num_experts: int = 0
n_routed_experts: int = 0
num_experts_per_tok: int = 0
# Expert MLP intermediate size (may differ from dense intermediate_size).
moe_intermediate_size: int = 0
moe_shared_expert_intermediate_size: int = 0
# Optional latent dimension used by Nemotron-H routed experts.
moe_latent_size: Optional[int] = None
routed_scaling_factor: float = 1.0
n_group: int = 1
topk_group: int = 1
# MoE layer frequency: layer (i+1) % decoder_sparse_step == 0 → MoE.
decoder_sparse_step: int = 1
# Layer indices that are always dense MLP (overrides decoder_sparse_step).
mlp_only_layers: List[int] = field(default_factory=list)
# Normalise top-k routing weights to sum to 1.
# Note: the C++ Int4MoePlugin hardcodes renormalize=true, so this field
# currently serves as documentation of the HF config value.
norm_topk_prob: bool = True
# Runtime vocabulary reduction. ``vocab_size`` remains the original
# tokenizer/embedding size; this field is only the exported logits size.
reduced_vocab_size: Optional[int] = None
# ------------------------------------------ per-layer embeddings (Gemma4)
# When > 0, Gemma4 E-model PLE is enabled. The ONNX graph receives
# a shared row pool and token-to-slot map, combined with the
# context-aware projection from inputs_embeds.
hidden_size_per_layer_input: int = 0
# Gemma4 E-model vocabulary for the runtime-side token-identity PLE table.
# Raw checkpoint rows are converted into the runtime pool.
vocab_size_per_layer_input: int = 0
# Gemma4: number of KV-shared layers (counted from the last layer backward).
# Layers [num_hidden_layers - num_kv_shared_layers, num_hidden_layers) are
# "KV-shared" — in HF they reuse KV states from the last non-shared layer
# of the same attention type. Our export gives them independent K/V, but
# they may have a 2× wider MLP (use_double_wide_mlp).
num_kv_shared_layers: int = 0
# When True, KV-shared layers use 2× intermediate_size for their MLP.
use_double_wide_mlp: bool = False
# Gemma4 26B-A4B MoE: when True, each decoder layer has a routed MoE
# block in addition to the dense MLP (parallel experts + router).
enable_moe_block: bool = False
# ------------------------------------------ tensor parallel
# ``mapping`` is the single source of truth for parallel placement.
# tp_size>1 returns a per-rank ONNX graph with col/row-parallel projections.
mapping: Mapping = field(default_factory=Mapping)
def __post_init__(self) -> None:
# For standalone models, num_kv_shared_layers < num_hidden_layers is
# required so that at least one non-shared donor layer exists.
# Gemma4 assistant models (shares_target_kv=True) share KV from the
# *target* model, so num_kv_shared_layers == num_hidden_layers is valid
# — the donor-index logic is skipped at runtime for those.
if (self.num_kv_shared_layers > 0
and self.num_kv_shared_layers >= self.num_hidden_layers
and not self.shares_target_kv):
raise ValueError(
f"num_kv_shared_layers ({self.num_kv_shared_layers}) must be "
f"less than num_hidden_layers ({self.num_hidden_layers})")
# Resolve and validate the model family's attention scale.
self.default_attention_scale = _get_attention_scaling(
{}, self.head_dim, self.default_attention_scale)
scale_config = ({} if self.attention_scaling is None else {
"attention_scaling": self.attention_scaling
})
self.attention_scaling = _get_attention_scaling(
scale_config, self.head_dim, self.default_attention_scale)
# ------------------------------------------------------------------
# Derived properties
# ------------------------------------------------------------------
@property
def tp_size(self) -> int:
return self.mapping.tp_size
@property
def tp_rank(self) -> int:
return self.mapping.tp_rank
@property
def is_eagle4_draft(self) -> bool:
"""An EAGLE-4 drafter: an EAGLE draft with ``fc_norm``, ``norm_output`` or a low-rank candidate head."""
return self.is_eagle3_draft and (self.eagle_fc_norm
or self.eagle_norm_output
or self.draft_candidate_count > 0)
@property
def is_eagle3_draft(self) -> bool:
return self.is_eagle3_draft_flag or (
self.draft_vocab_size is not None and not self.is_dflash_draft
and not self.dflash_base and not self.is_jetspec_draft
and not self.jetspec_base and not self.is_dspark_draft
and not self.dspark_base)
@property
def is_mtp_draft(self) -> bool:
"""True for a derived MTP draft config built from a base checkpoint."""
if self.mtp_base or self.is_eagle3_draft or self.is_dflash_draft \
or self.is_jetspec_draft or self.is_dspark_draft:
return False
return self.is_mtp_draft_flag
@property
def is_gemma4_mtp_draft(self) -> bool:
"""True for a paired Gemma4 assistant draft checkpoint."""
return self.gemma4_mtp_draft
@property
def is_diffusion_gemma(self) -> bool:
return self.diffusion is not None or _is_diffusion_gemma_model_type(
self.model_type)
@property
def is_dflash_draft(self) -> bool:
return self.is_dflash_draft_flag
@property
def is_domino_draft(self) -> bool:
return (self.is_dflash_draft_flag
and self.dflash_projector_type == "domino")
@property
def domino_base(self) -> bool:
return self.dflash_base and self.dflash_projector_type == "domino"
@property
def is_jetspec_draft(self) -> bool:
return self.is_jetspec_draft_flag
@property
def is_dspark_draft(self) -> bool:
return self.is_dspark_draft_flag
@property
def ple_enabled(self) -> bool:
"""True when Gemma4 per-layer embeddings are enabled."""
return self.hidden_size_per_layer_input > 0
@property
def ngram_layers(self) -> List[int]:
"""Zero-indexed decoder layers carrying a Qwen4-Exp n-gram injection.
Unrelated to :attr:`ple_enabled`, which is Gemma4's per-layer embeddings; the two features
share only a name.
"""
return sorted(index - 1 for index in self.ple_layer_ids)
@property
def ngram_heads(self) -> int:
"""Row IDs looked up per token: one per hash head, over all n-gram orders but unigram."""
return (self.ngram_size - 1) * self.heads_per_ngram
@property
def eagle3_target_hidden_size(self) -> int:
return self.target_hidden_size or self.hidden_size
@property
def eagle3_num_target_layers(self) -> int:
return len(self.eagle3_target_layer_ids) or 3
@property
def is_hybrid(self) -> bool:
return self.mamba_cfg is not None or self.gdn_cfg is not None
@property
def is_nemotron_h(self) -> bool:
return (self.model_type or "").lower().startswith("nemotron_h")
@property
def num_attn_layers(self) -> int:
"""Total attention layers (includes Gemma4 sliding/full variants).
Note: layers may have heterogeneous head dims — use per-layer configs
(kv_layer_configs) for allocation, not this count alone.
"""
return sum(1 for t in self.layer_types
if t in (LAYER_ATTN, "sliding_attention", "full_attention"))
@property
def kv_pool_head_dim(self) -> int:
"""Paged KV pool row width of an attention layer.
A checkpoint with an indexer (``indexer_head_dim > 0``) runs every
attention layer as Qwen Sparse Attention, and QsaAttentionPlugin
persists the indexer state in a tail of ``indexer_head_dim`` columns
after the ``head_dim`` K/V columns; without an indexer the row is
``head_dim`` (Gemma4's global layers are sized by their own per-layer
hook). The exported ``kv_layer_configs`` and a model's
``past_key_values`` export spec must both size the pool with this,
while the top-level ``head_dim`` (which sizes the rotary binding)
stays untouched.
"""
return self.head_dim + self.indexer_head_dim
@property
def num_mamba_layers(self) -> int:
return sum(1 for t in self.layer_types if t == LAYER_MAMBA)
@property
def num_gdn_layers(self) -> int:
return sum(1 for t in self.layer_types if t == LAYER_GDN)
@property
def num_mlp_layers(self) -> int:
return sum(1 for t in self.layer_types if t == LAYER_MLP)
@property
def num_moe_layers(self) -> int:
return sum(1 for t in self.layer_types if t == LAYER_MOE)
@property
def use_dual_rope(self) -> bool:
return (self.sliding_rope_config is not None
and self.full_rope_config is not None)
@property
def compute_dtype(self) -> "torch.dtype": # noqa: F821
import torch
_MAP = {
"bfloat16": torch.bfloat16,
"float16": torch.float16,
"float32": torch.float32,
}
return _MAP.get(self.torch_dtype, torch.bfloat16)
[docs]
def for_rank(self, rank: int, world: int) -> "ModelConfig":
"""Return a per-rank copy of this config for TP.
Divides head and intermediate sizes by *world* so each rank's model
carries per-rank shapes.
Usage:
cfg = ModelConfig.from_pretrained(
path, lambda head_dim: 1.0 / (float(head_dim)**0.5)
).for_rank(rank, world)
model = CausalLM(cfg)
load_weights(model, path, mapping=cfg.mapping)
"""
import copy
if world == 1:
return self
parallel_dimensions = [
("num_attention_heads", self.num_attention_heads),
("num_key_value_heads", self.num_key_value_heads),
("intermediate_size", self.intermediate_size),
]
if self.gdn_cfg is not None:
parallel_dimensions.extend((
("gdn_cfg.num_key_heads", self.gdn_cfg.num_key_heads),
("gdn_cfg.num_value_heads", self.gdn_cfg.num_value_heads),
))
for name, v in parallel_dimensions:
if v % world:
raise ValueError(
f"TP world={world}: {name}={v} is not divisible by {world}"
)
c = copy.deepcopy(self)
c.mapping = Mapping(world_size=world,
rank=rank,
tp_size=world,
tp_rank=rank)
c.num_attention_heads //= world
c.num_key_value_heads //= world
c.intermediate_size //= world
if c.gdn_cfg is not None:
c.gdn_cfg.num_key_heads //= world
c.gdn_cfg.num_value_heads //= world
return c
# ------------------------------------------------------------------
# Factory
# ------------------------------------------------------------------
[docs]
@classmethod
def from_pretrained(
cls, model_dir: str,
default_attention_scale: Callable[[int], float]) -> "ModelConfig":
"""Load a ModelConfig from a checkpoint directory.
Loads architecture hyper-parameters via ``AutoConfig`` (see
:func:`checkpoint_utils.load_checkpoint_config_dicts`) and then
either ``hf_quant_config.json`` or the embedded ``quantization_config``
block to determine
the quantisation scheme.
``default_attention_scale`` is a required model-family callable
accepting ``head_dim``. ``has_qk_norm`` is auto-detected by scanning
the safetensors key index for ``.q_norm.weight`` entries; the HunYuan
``.query_layernorm.weight`` spelling is honored only for model types
whose loader remaps it (see :func:`_detect_has_qk_norm`).
"""
root, llm_dict = load_checkpoint_config_dicts(model_dir)
if llm_dict.get("num_hidden_layers") is None:
layer_schedule = (llm_dict.get("layers_block_type")
or llm_dict.get("layer_types"))
if isinstance(layer_schedule, (list, tuple)) and layer_schedule:
llm_dict = dict(llm_dict)
llm_dict["num_hidden_layers"] = len(layer_schedule)
root_model_type = root.get("model_type", "")
model_type = llm_dict.get("model_type", "llama")
is_diffusion_gemma = _is_diffusion_gemma_config(root, llm_dict)
is_gemma4_assistant = _is_gemma4_assistant_model_type(root_model_type)
if is_gemma4_assistant:
model_type = "gemma4_assistant"
elif is_diffusion_gemma:
model_type = "diffusion_gemma"
hidden_size = llm_dict["hidden_size"]
num_attn_heads = llm_dict["num_attention_heads"]
_check_num_attention_heads(num_attn_heads)
head_dim = llm_dict.get("head_dim", hidden_size // num_attn_heads)
global_head_dim = int(llm_dict.get("global_head_dim", 0) or 0)
num_global_kv_heads = int(
llm_dict.get("num_global_key_value_heads", 0) or 0)
indexer_head_dim = int(llm_dict.get("indexer_head_dim", 0) or 0)
# Omni roots keep one ignore list for thinker + talker; scope it to the
# sub-model this config represents (talker/CP go through their own
# staged sub-config instead).
submodel_prefix = ""
if root.get("thinker_config") is not None and llm_dict is not root:
submodel_prefix = "thinker."
quant_dict = llm_dict
if ("quantization_config" not in quant_dict
and root.get("quantization_config") is not None):
quant_dict = dict(llm_dict)
quant_dict["quantization_config"] = root["quantization_config"]
quant = _parse_quant(model_dir, quant_dict, submodel_prefix)
raw_layer_types = _parse_raw_layer_types(llm_dict)
layer_types = _parse_layer_types(llm_dict)
block_configs = _parse_puzzle_block_configs(llm_dict, layer_types)
attention_layer_types = _parse_attention_layer_types(
llm_dict, llm_dict["num_hidden_layers"], model_type)
num_kv_heads = llm_dict.get("num_key_value_heads", num_attn_heads)
full_attention_only = (_is_gemma4_model_type(model_type)
and attention_layer_types and all(
layer_type == "full_attention"
for layer_type in attention_layer_types))
if full_attention_only:
if global_head_dim:
head_dim = global_head_dim
if num_global_kv_heads:
num_kv_heads = num_global_kv_heads
dual_rope_configs = _get_dual_rope_configs(llm_dict)
mamba_cfg = _parse_mamba_cfg(llm_dict,
layer_types,
model_dir=model_dir)
gdn_cfg = _parse_gdn_cfg(llm_dict, layer_types)
has_qk_norm = _detect_has_qk_norm(model_dir, model_type)
has_value_norm = _get_has_value_norm(llm_dict, model_type)
default_attention_scale_value = float(
default_attention_scale(head_dim))
attention_scaling = _get_attention_scaling(
llm_dict, head_dim, default_attention_scale_value)
embedding_scale = _get_embedding_scale(llm_dict, model_type,
hidden_size)
if _is_muse_glimmer_model_type(model_type):
qk_scale_factor = llm_dict.get("qk_scale_factor")
if qk_scale_factor is not None:
attention_scaling = float(qk_scale_factor) / math.sqrt(
head_dim)
# Full-attention layers use NoPE; sliding layers use regular RoPE.
if not dual_rope_configs:
_mp = int(llm_dict.get("max_position_embeddings", 4096))
_sliding = {
"rope_theta": _get_rope_theta(llm_dict),
"rope_scaling": None,
"partial_rotary_factor": 1.0,
"max_position_embeddings": _mp,
}
_full = dict(_sliding)
_full["rope_scaling"] = {"rope_type": "nope", "type": "nope"}
dual_rope_configs = {
"sliding_rope_config": _sliding,
"full_rope_config": _full,
}
generation_config_path = os.path.join(model_dir,
"generation_config.json")
generation_config: Dict[str, Any] = {}
if os.path.isfile(generation_config_path):
with open(generation_config_path) as f:
generation_config = json.load(f)
diffusion_source_config = dict(root)
diffusion_source_config.update(llm_dict)
diffusion_config = (DiffusionConfig.from_hf(diffusion_source_config,
generation_config)
if is_diffusion_gemma else None)
self_conditioning_size = 0
# MTP config
mtp_num_hidden_layers = llm_dict.get("mtp_num_hidden_layers")
if mtp_num_hidden_layers is not None:
mtp_num_hidden_layers = int(mtp_num_hidden_layers)
mtp_hybrid_override_pattern = llm_dict.get(
"mtp_hybrid_override_pattern")
mtp_layer_types = _parse_mtp_layer_types(llm_dict)
num_nextn_predict_layers = int(
llm_dict.get("num_nextn_predict_layers", 0) or 0)
if (mtp_num_hidden_layers is None and num_nextn_predict_layers > 0
and mtp_layer_types):
mtp_num_hidden_layers = len(mtp_layer_types)
elif (mtp_num_hidden_layers is None and num_nextn_predict_layers > 0
and model_type in _MTP_CONFIG_MODEL_TYPES):
# Dense Qwen3 MTP heads (e.g. AngelSlim/Qwen3-8B-MTP) declare only
# the DeepSeek-style prediction-module count.
mtp_num_hidden_layers = num_nextn_predict_layers
mtp_use_dedicated_embeddings = bool(
llm_dict.get("mtp_use_dedicated_embeddings", False))
_validate_mtp_constraints(
model_type=model_type,
mtp_num_hidden_layers=mtp_num_hidden_layers,
mtp_use_dedicated_embeddings=mtp_use_dedicated_embeddings,
)
# EAGLE3 draft model fields
speculator_type = llm_dict.get("speculators_model_type", "")
if speculator_type == "peagle":
raise NotImplementedError(
"P-EAGLE uses a parallel drafting contract that is not supported "
"by the Edge-LLM EAGLE3 runtime.")
architectures = llm_dict.get("architectures", []) or []
is_eagle3_draft_flag = any(
"Eagle3" in str(arch) for arch in architectures) or (
"ttt_length" in llm_dict) or speculator_type == "eagle3"
has_eagle4_features = bool(_eagle_draft_feature_keys(llm_dict))
if has_eagle4_features:
# Drafts with fc_norm, norm_output or a candidate head also mark
# themselves with an Eagle4 architecture or TorchSpec's
# eagle_config.use_aux_hidden_state.
is_eagle3_draft_flag = is_eagle3_draft_flag or any(
"Eagle4" in str(arch) for arch in architectures) or bool(
(llm_dict.get("eagle_config")
or {}).get("use_aux_hidden_state"))
draft_vocab_size = llm_dict.get("draft_vocab_size", None)
target_hidden_size = llm_dict.get("target_hidden_size", None)
eagle3_target_layer_ids = list(
llm_dict.get("target_layer_ids", [])
or []) if is_eagle3_draft_flag else []
eagle_features = _EagleDraftFeatures()
if is_eagle3_draft_flag and has_eagle4_features:
eagle3_target_layer_ids = _parse_eagle_target_layer_ids(llm_dict)
eagle_features = _parse_eagle_draft_features(
llm_dict, eagle3_target_layer_ids)
if is_eagle3_draft_flag:
draft_vocab_size = draft_vocab_size or llm_dict.get("vocab_size")
target_hidden_size = target_hidden_size or hidden_size
# Sliding window: active when use_sliding_window=True, or when
# layer_types contains "sliding_attention" (Gemma4 convention).
use_sw = llm_dict.get("use_sliding_window", False) or any(
layer_type == "sliding_attention"
for layer_type in raw_layer_types)
sw_raw = llm_dict.get("sliding_window") if use_sw else None
sliding_window_size = int(sw_raw) if sw_raw is not None else -1
skip_softmax_prefill_enabled = bool(
llm_dict.get("skip_softmax_prefill_enabled", False))
skip_softmax_decode_enabled = bool(
llm_dict.get("skip_softmax_decode_enabled", False))
use_vision_bidirectional_attention = bool(
model_type in ("gemma4_unified", "gemma4_unified_text")
and llm_dict.get("use_bidirectional_attention") == "vision")
jetspec_config = (llm_dict.get("jetspec_config")
or llm_dict.get("dflash_config") or {})
jetspec_target_layer_ids = list(
jetspec_config.get("target_layer_ids",
llm_dict.get("target_layer_ids", [])) or [])
jetspec_block_size = int(
jetspec_config.get("block_size", llm_dict.get("block_size", 16)))
jetspec_mask_token_id = int(
jetspec_config.get("mask_token_id",
llm_dict.get("mask_token_id", 151669)))
jetspec_causal_head = bool(
jetspec_config.get("causal_head",
llm_dict.get("causal_head", True)))
# Sparse MoE fields. HF uses "num_local_experts" as the internal key
# and maps "num_experts" → "num_local_experts" via attribute_map.
num_experts = int(
llm_dict.get("num_experts", llm_dict.get("num_local_experts", 0))
or 0)
# Nemotron-H declares routed experts as ``n_routed_experts`` rather
# than ``num_experts`` / ``num_local_experts``; treat either as MoE so
# the sizes below are parsed instead of defaulting to 0.
n_routed_experts = int(llm_dict.get("n_routed_experts", 0) or 0)
if num_experts > 0 or n_routed_experts > 0:
num_experts_per_tok = int(
llm_dict.get("num_experts_per_tok",
llm_dict.get("top_k_experts", 0)) or 0)
moe_intermediate_size = int(
llm_dict.get("moe_intermediate_size", 0) or 0)
else:
num_experts_per_tok = 0
moe_intermediate_size = 0
# HF Qwen3-Omni MoE Talker uses the un-prefixed name; HF NemotronH /
# other MoE families use ``moe_shared_expert_intermediate_size``.
moe_shared_expert_intermediate_size = int(
llm_dict.get("moe_shared_expert_intermediate_size",
llm_dict.get("shared_expert_intermediate_size", 0))
or 0)
routed_scaling_factor = float(
llm_dict.get("routed_scaling_factor", 1.0))
n_group = int(llm_dict.get("n_group", 1))
topk_group = int(llm_dict.get("topk_group", 1))
decoder_sparse_step = int(llm_dict.get("decoder_sparse_step", 1))
mlp_only_layers = list(llm_dict.get("mlp_only_layers") or [])
norm_topk_prob = bool(llm_dict.get("norm_topk_prob", True))
moe_latent_size = llm_dict.get("moe_latent_size", None)
if moe_latent_size is not None:
moe_latent_size = int(moe_latent_size)
intermediate_size = int(
llm_dict.get("intermediate_size")
or llm_dict.get("shared_expert_intermediate_size")
or llm_dict.get("moe_shared_expert_intermediate_size")
or llm_dict.get("moe_intermediate_size", 0))
if self_conditioning_size <= 0:
self_conditioning_size = int(
llm_dict.get(
"self_conditioning_size",
root.get("self_conditioning_size", intermediate_size))
or intermediate_size)
config = cls(
model_type=model_type,
hidden_size=hidden_size,
num_hidden_layers=llm_dict["num_hidden_layers"],
num_attention_heads=num_attn_heads,
num_key_value_heads=num_kv_heads,
intermediate_size=intermediate_size,
head_dim=head_dim,
global_head_dim=global_head_dim,
num_global_key_value_heads=num_global_kv_heads,
indexer_head_dim=indexer_head_dim,
rms_norm_eps=_get_rms_norm_eps(llm_dict, model_type),
vocab_size=llm_dict["vocab_size"],
rope_theta=_get_rope_theta(llm_dict),
max_position_embeddings=llm_dict.get("max_position_embeddings",
4096),
default_attention_scale=default_attention_scale_value,
rope_scaling=_select_rope_scaling(llm_dict),
original_max_position_embeddings=llm_dict.get(
"original_max_position_embeddings", None),
partial_rotary_factor=_get_partial_rotary_factor(llm_dict),
hidden_activation=llm_dict.get("hidden_activation",
llm_dict.get("hidden_act", "silu")),
num_code_groups=int(llm_dict.get("num_code_groups", 0) or 0),
sliding_rope_config=dual_rope_configs.get("sliding_rope_config"),
full_rope_config=dual_rope_configs.get("full_rope_config"),
has_qk_norm=has_qk_norm,
qk_norm_post_rope=(model_type in _QK_NORM_POST_ROPE_MODEL_TYPES),
has_value_norm=has_value_norm,
attention_bias=bool(llm_dict.get("attention_bias", False)),
attention_scaling=attention_scaling,
attention_k_eq_v=(True if is_diffusion_gemma else bool(
llm_dict.get("attention_k_eq_v", False))),
encoder_layer_scalars=list(
llm_dict.get("encoder_layer_scalars",
root.get("encoder_layer_scalars", [])) or []),
decoder_layer_scalars=list(
llm_dict.get("decoder_layer_scalars",
root.get("decoder_layer_scalars", [])) or []),
self_conditioning_size=self_conditioning_size,
diffusion=diffusion_config,
embedding_scale=embedding_scale,
final_logit_softcapping=llm_dict.get("final_logit_softcapping",
None),
output_multiplier=float(llm_dict.get("output_multiplier", 1.0)),
post_norm_eps=(float(llm_dict["post_norm_eps"])
if llm_dict.get("post_norm_eps") is not None else
None),
torch_dtype=llm_dict.get("torch_dtype",
llm_dict.get("dtype", "bfloat16")),
tie_word_embeddings=llm_dict.get("tie_word_embeddings", False),
sliding_window_size=sliding_window_size,
skip_softmax_prefill_enabled=skip_softmax_prefill_enabled,
skip_softmax_decode_enabled=skip_softmax_decode_enabled,
use_vision_bidirectional_attention=
use_vision_bidirectional_attention,
layer_types=layer_types,
block_configs=block_configs,
attention_layer_types=attention_layer_types,
quant=quant,
mamba_cfg=mamba_cfg,
gdn_cfg=gdn_cfg,
attn_output_gate=bool(llm_dict.get("attn_output_gate", False)),
hc_count=int(llm_dict.get("hc_count", 0)),
hc_lowrank=int(llm_dict.get("hc_lowrank", 0)),
indexer_n_heads=int(llm_dict.get("indexer_n_heads") or 0),
indexer_kv_heads=int(llm_dict.get("indexer_kv_heads") or 0),
indexer_budget=int(llm_dict.get("indexer_budget") or 0),
indexer_compress_ratio=int(
llm_dict.get("indexer_compress_ratio") or 0),
ple_layer_ids=[
int(i) for i in (llm_dict.get("ple_layer_ids") or [])
],
ple_embed_dim=int(llm_dict.get("ple_embed_dim", 0)),
ple_conv_kernel_size=int(llm_dict.get("ple_conv_kernel_size", 0)),
ngram_size=int(llm_dict.get("ngram_size", 0)),
heads_per_ngram=int(llm_dict.get("heads_per_ngram", 0)),
split_ngram_parts=int(llm_dict.get("split_ngram_parts") or 0),
mtp_num_hidden_layers=mtp_num_hidden_layers,
mtp_use_dedicated_embeddings=mtp_use_dedicated_embeddings,
mtp_hybrid_override_pattern=mtp_hybrid_override_pattern,
mtp_layer_types=mtp_layer_types,
mtp_base=bool(llm_dict.get("mtp_base", False)),
root_model_type=root_model_type,
raw_layer_types=raw_layer_types,
rope_parameters=llm_dict.get("rope_parameters", None),
backbone_hidden_size=int(
llm_dict.get("backbone_hidden_size", 0) or 0),
assistant_hidden_size=(hidden_size if is_gemma4_assistant else 0),
shares_target_kv=is_gemma4_assistant,
has_own_kv_cache=not is_gemma4_assistant,
constant_draft_positions=is_gemma4_assistant,
returns_feedback_hidden=is_gemma4_assistant,
use_ordered_embeddings=bool(
llm_dict.get("use_ordered_embeddings", False)),
num_centroids=int(llm_dict.get("num_centroids", 0) or 0),
centroid_intermediate_top_k=int(
llm_dict.get("centroid_intermediate_top_k", 0) or 0),
dflash_base=bool(llm_dict.get("dflash_base", False)),
jetspec_base=bool(llm_dict.get("jetspec_base", False)),
jetspec_target_layer_ids=jetspec_target_layer_ids,
jetspec_block_size=jetspec_block_size,
jetspec_mask_token_id=jetspec_mask_token_id,
jetspec_causal_head=jetspec_causal_head,
dflash_target_layer_ids=list(
(llm_dict.get("dflash_config", {})
or {}).get("target_layer_ids")
or llm_dict.get("eagle_aux_hidden_state_layer_ids") or []),
dspark_base=bool(llm_dict.get("dspark_base", False)),
num_deepstack_features=_parse_num_deepstack_features(
llm_dict, model_type, root_config=root),
accept_hidden_layer=_parse_accept_hidden_layer(llm_dict,
root_config=root),
draft_vocab_size=draft_vocab_size,
target_hidden_size=target_hidden_size,
is_eagle3_draft_flag=is_eagle3_draft_flag,
eagle3_target_layer_ids=eagle3_target_layer_ids,
eagle3_norm_before_residual=bool(
llm_dict.get("norm_before_residual", False)),
eagle3_norm_before_fc=bool(llm_dict.get("norm_before_fc", False)),
eagle_fc_norm=eagle_features.fc_norm,
eagle_norm_output=eagle_features.norm_output,
draft_candidate_rank=eagle_features.candidate_rank,
draft_candidate_count=eagle_features.candidate_count,
num_experts=num_experts,
n_routed_experts=n_routed_experts,
num_experts_per_tok=num_experts_per_tok,
moe_intermediate_size=moe_intermediate_size,
moe_shared_expert_intermediate_size=
moe_shared_expert_intermediate_size,
moe_latent_size=moe_latent_size,
routed_scaling_factor=routed_scaling_factor,
n_group=n_group,
topk_group=topk_group,
decoder_sparse_step=decoder_sparse_step,
mlp_only_layers=mlp_only_layers,
norm_topk_prob=norm_topk_prob,
hidden_size_per_layer_input=int(
llm_dict.get("hidden_size_per_layer_input", 0) or 0),
vocab_size_per_layer_input=int(
llm_dict.get("vocab_size_per_layer_input", 0) or 0),
num_kv_shared_layers=int(
llm_dict.get("num_kv_shared_layers", 0) or 0),
use_double_wide_mlp=bool(llm_dict.get("use_double_wide_mlp",
False)),
enable_moe_block=bool(
llm_dict.get(
"enable_moe_block", model_type in (
"gemma4",
"gemma4_text",
"gemma4_unified",
"gemma4_unified_text",
"diffusion_gemma",
) and num_experts > 0 and moe_intermediate_size > 0)),
)
if has_eagle4_features and not is_eagle3_draft_flag \
and config.is_eagle3_draft:
raise ValueError(
"This EAGLE draft config sets "
f"{_describe_eagle_draft_features(llm_dict)} but no EAGLE "
"draft marker (an Eagle3 or Eagle4 architecture, ttt_length or "
"eagle_config.use_aux_hidden_state), so it would export as a "
"dense EAGLE3 draft without them. Add the draft's Eagle4 "
"architecture to its config.json.")
return config
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
# When HF omits ``deepstack_visual_indexes`` / ``num_deepstack_features``,
# these model_types are known to expect three visual deepstack injections
# at runtime (Thinker side only). Listed explicitly — substring matching
# (``"qwen3_omni" in "qwen3_omni_moe_talker"``) would otherwise mis-classify
# Talker / CodePredictor configs as deepstack producers and bake 3 dangling
# input ports into their engines.
_DEEPSTACK_MODEL_TYPES = frozenset({
# HF root configs that wrap a deepstack-emitting visual encoder
"qwen3_vl",
"qwen3_omni",
"qwen3_omni_moe",
# Standalone Thinker text-LLM configs (after quant export); the Thinker
# still consumes deepstack inputs from the separately exported visual
# encoder at runtime.
"qwen3_vl_text",
"qwen3_omni_text",
"qwen3_omni_moe_text",
})
# Model types whose checkpoints may carry a Qwen3-Next-layout ``mtp.*`` head.
# Dense ``qwen3`` is served by the same draft class with plain decoder layers
# (validated with AngelSlim/Qwen3-8B-MTP); add a family only with a validated head.
_MTP_CONFIG_MODEL_TYPES = frozenset({
"qwen3",
"qwen3_5",
"qwen3_5_text",
"qwen3_5_moe",
"qwen3_5_moe_text",
"qwen3_omni_next_text",
"qwen3_omni_next_text_moe",
"qwen4_exp_text",
})
# Qwen3.5-family model types (checkpoint and text-config) whose hybrid base
# exports the EAGLE verify ABI. Qwen3-Omni-Next reuses the Qwen3.5 classes but
# is not one of them.
_QWEN35_EAGLE_BASE_MODEL_TYPES = frozenset({
"qwen3_5",
"qwen3_5_text",
"qwen3_5_moe",
"qwen3_5_moe_text",
})
def make_mtp_draft_config(base_config: ModelConfig) -> ModelConfig:
"""Derive the currently supported MTP draft config from a base config."""
_validate_mtp_constraints(
model_type=base_config.model_type,
mtp_num_hidden_layers=base_config.mtp_num_hidden_layers,
mtp_use_dedicated_embeddings=base_config.mtp_use_dedicated_embeddings,
)
mtp_num_hidden_layers = base_config.mtp_num_hidden_layers
if mtp_num_hidden_layers is None:
raise ValueError(
"MTP draft config requires mtp_num_hidden_layers in the base config."
)
# Nemotron-H: Exclude all draft ``layers.*`` modules from quantization;
# the untouched lm_head keeps the base quant type.
if (base_config.model_type or "").lower().startswith("nemotron_h"):
draft_layer_types = list(base_config.mtp_layer_types)
if len(draft_layer_types) != mtp_num_hidden_layers:
declared = (base_config.mtp_hybrid_override_pattern
or base_config.mtp_layer_types)
raise ValueError(
f"MTP draft stack {declared!r} yields "
f"{len(draft_layer_types)} layers != mtp_num_hidden_layers "
f"{mtp_num_hidden_layers}")
draft_quant = replace(
base_config.quant,
excluded=list(base_config.quant.excluded) + ["layers.*"],
)
return replace(
base_config,
num_hidden_layers=mtp_num_hidden_layers,
layer_types=draft_layer_types,
mamba_cfg=None,
gdn_cfg=None,
mtp_base=False,
is_mtp_draft_flag=True,
quant=draft_quant,
tie_word_embeddings=False,
)
# The draft is quantized iff its FFN compute weights are quantized
# (routed experts for MoE, dense MLP otherwise) — other modules may be
# excluded in any recipe. Excluded entries are exact names or globs.
ffn_probe = ("mtp.layers.0.mlp.experts.0.gate_proj" if
base_config.num_experts > 0 else "mtp.layers.0.mlp.gate_proj")
mtp_is_quantized = not any(
fnmatch.fnmatch(ffn_probe, e) for e in base_config.quant.excluded)
if mtp_is_quantized:
# MTP draft modules are independently quantized. Strip base-model
# layer_overrides that use module paths absent from the draft
# (e.g. "model.layers.0.linear_attn.in_proj_qkv") — they would
# cause spurious FP16 fallback for draft-specific layers like "fc".
# Keep entries whose keys also exist in the draft module namespace
# (e.g. "lm_head") so that lm_head_quantization is honoured.
_DRAFT_MODULE_PREFIXES = ("lm_head", "fc", "layers.", "norm")
def _draft_entries(entries: dict) -> dict:
# ``mtp.``-prefixed entries name draft modules and win over
# same-named base entries.
kept = {
k: v
for k, v in entries.items() if any(
k == p or k.startswith(p) for p in _DRAFT_MODULE_PREFIXES)
}
kept.update({
k[len("mtp."):]: v
for k, v in entries.items() if k.startswith("mtp.")
})
return kept
draft_overrides = _draft_entries(base_config.quant.layer_overrides)
draft_group_sizes = _draft_entries(base_config.quant.layer_group_sizes)
draft_listed = any(
k.startswith("mtp.") for k in base_config.quant.layer_overrides)
# Preserve MTP-specific exclusions (e.g. mtp.lm_head when lm_head
# is FP16) but drop base-model exclusions irrelevant to the draft.
draft_excluded = [
e[len("mtp."):] for e in base_config.quant.excluded
if e.startswith("mtp.")
]
# A checkpoint that lists draft modules keeps mixed precision
# (unlisted draft modules are FP16). One that lists none sends
# unlisted modules (fc, q_proj, etc.) to the dominant quant type;
# explicit overrides (lm_head→fp8) still apply.
draft_quant = replace(
base_config.quant,
excluded=draft_excluded,
layer_overrides=draft_overrides,
layer_group_sizes=draft_group_sizes,
is_mixed_precision=(base_config.quant.is_mixed_precision
and draft_listed))
else:
# MTP draft lm_head is borrowed from the base model and may itself be quantized.
lm_head_overrides = {
k: v
for k, v in base_config.quant.layer_overrides.items()
if k == "lm_head" or k.startswith("lm_head.")
}
lm_head_group_sizes = {
k: v
for k, v in base_config.quant.layer_group_sizes.items()
if k == "lm_head" or k.startswith("lm_head.")
}
draft_quant = QuantConfig(layer_overrides=lm_head_overrides,
layer_group_sizes=lm_head_group_sizes)
return replace(
base_config,
num_hidden_layers=mtp_num_hidden_layers,
layer_types=[LAYER_ATTN] * mtp_num_hidden_layers,
gdn_cfg=None,
ple_layer_ids=[],
mtp_base=False,
is_mtp_draft_flag=True,
quant=draft_quant,
tie_word_embeddings=False,
)
def make_dspark_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float]) -> ModelConfig:
"""Build a DSpark draft ModelConfig from a DeepSpec DSpark checkpoint."""
_, llm_dict = load_checkpoint_config_dicts(draft_dir)
dspark_config = (llm_dict.get("dspark_config")
or llm_dict.get("dflash_config", {}) or {})
quant = _parse_quant(draft_dir, llm_dict)
target_layer_ids = list(
dspark_config.get("target_layer_ids",
llm_dict.get("target_layer_ids", [])))
if not target_layer_ids:
raise ValueError(
"DSpark draft config requires target_layer_ids in config.json.")
model_type = llm_dict.get("model_type", "qwen3")
raw_layer_types = _parse_raw_layer_types(llm_dict)
layer_types = _parse_layer_types(llm_dict)
attention_layer_types = _parse_attention_layer_types(
llm_dict, llm_dict["num_hidden_layers"], model_type)
if (not attention_layer_types and raw_layer_types
and all(layer_type in _VALID_ATTENTION_LAYER_TYPES
for layer_type in raw_layer_types)):
attention_layer_types = list(raw_layer_types)
full_attention_only = (_is_gemma4_model_type(model_type)
and attention_layer_types
and all(layer_type == "full_attention"
for layer_type in attention_layer_types))
head_dim = llm_dict.get(
"global_head_dim" if full_attention_only else "head_dim",
llm_dict.get(
"head_dim",
llm_dict["hidden_size"] // llm_dict["num_attention_heads"]))
global_head_dim = int(llm_dict.get("global_head_dim", 0) or 0)
num_global_kv_heads = int(
llm_dict.get("num_global_key_value_heads", 0) or 0)
num_kv_heads = (
num_global_kv_heads if full_attention_only and num_global_kv_heads else
llm_dict.get("num_key_value_heads", llm_dict["num_attention_heads"]))
dual_rope_configs = _get_dual_rope_configs(llm_dict)
use_sw = llm_dict.get("use_sliding_window", False) or any(
layer_type == "sliding_attention" for layer_type in raw_layer_types)
sw_raw = llm_dict.get("sliding_window") if use_sw else None
sliding_window_size = int(sw_raw) if sw_raw is not None else -1
default_attention_scale_value = float(default_attention_scale(head_dim))
default_mask_token_id = 4 if _is_gemma4_model_type(model_type) else 151669
draft_vocab_size = llm_dict.get("draft_vocab_size")
draft_vocab_size = int(llm_dict["vocab_size"] if draft_vocab_size is
None else draft_vocab_size)
return ModelConfig(
model_type=model_type,
hidden_size=llm_dict["hidden_size"],
num_hidden_layers=llm_dict["num_hidden_layers"],
num_attention_heads=llm_dict["num_attention_heads"],
num_key_value_heads=num_kv_heads,
intermediate_size=llm_dict["intermediate_size"],
head_dim=head_dim,
global_head_dim=global_head_dim,
num_global_key_value_heads=num_global_kv_heads,
rms_norm_eps=llm_dict.get("rms_norm_eps", 1e-6),
vocab_size=llm_dict["vocab_size"],
rope_theta=_get_rope_theta(llm_dict),
max_position_embeddings=llm_dict.get("max_position_embeddings", 4096),
default_attention_scale=default_attention_scale_value,
rope_scaling=_select_rope_scaling(llm_dict),
partial_rotary_factor=_get_partial_rotary_factor(llm_dict),
hidden_activation=llm_dict.get("hidden_activation",
llm_dict.get("hidden_act", "silu")),
sliding_rope_config=dual_rope_configs.get("sliding_rope_config"),
full_rope_config=dual_rope_configs.get("full_rope_config"),
has_qk_norm=True,
has_value_norm=_get_has_value_norm(llm_dict, model_type),
attention_bias=bool(llm_dict.get("attention_bias", False)),
attention_scaling=_get_attention_scaling(
llm_dict, head_dim, default_attention_scale_value),
attention_k_eq_v=bool(llm_dict.get("attention_k_eq_v", False)),
attention_sink_bias=bool(
dspark_config.get("attention_sink_bias",
llm_dict.get("attention_sink_bias", False))),
final_logit_softcapping=llm_dict.get("final_logit_softcapping", None),
torch_dtype=llm_dict.get("torch_dtype",
llm_dict.get("dtype", "bfloat16")),
tie_word_embeddings=False,
sliding_window_size=sliding_window_size,
layer_types=layer_types,
attention_layer_types=attention_layer_types,
raw_layer_types=raw_layer_types,
rope_parameters=llm_dict.get("rope_parameters", None),
draft_vocab_size=draft_vocab_size,
reduced_vocab_size=(draft_vocab_size if draft_vocab_size
< llm_dict["vocab_size"] else None),
is_dspark_draft_flag=True,
dspark_causal_proposal=bool(
dspark_config.get(
"causal",
llm_dict.get("dflash_query_causal",
(llm_dict.get("dflash_config")
or {}).get("causal", False)))),
dspark_target_layer_ids=target_layer_ids,
dspark_block_size=int(
dspark_config.get("block_size", llm_dict.get("block_size", 7))),
dspark_mask_token_id=int(
dspark_config.get(
"mask_token_id",
llm_dict.get("mask_token_id", default_mask_token_id))),
dspark_enable_confidence_head=bool(
dspark_config.get("enable_confidence_head",
llm_dict.get("enable_confidence_head", False))),
dspark_sample_from_anchor=bool(
dspark_config.get("sample_from_anchor",
llm_dict.get("sample_from_anchor", True))),
dspark_confidence_head_with_markov=bool(
dspark_config.get(
"confidence_head_with_markov",
llm_dict.get("confidence_head_with_markov", False))),
dspark_markov_head_type=str(
dspark_config.get("markov_head_type",
llm_dict.get("markov_head_type", ""))),
dspark_markov_rank=int(
dspark_config.get("markov_rank", llm_dict.get("markov_rank", 0))),
quant=quant,
)
def make_dflash_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float],
target_vocab_size: Optional[int] = None,
quantize_w4a16_activations: bool = False) -> ModelConfig:
"""Build a DFlash draft ModelConfig from the draft checkpoint directory.
``quantize_w4a16_activations`` runs a declared W4A16 NVFP4 draft as W4A4 for
the modules that ship an input scale.
Now quantization-aware: if the draft directory contains
``hf_quant_config.json`` (e.g. from DFlash draft NVFP4 quantization),
the quant config is parsed so ``make_linear`` will produce the correct
Linear class (``NVFP4Linear`` etc.) during ONNX export.
"""
root_dict, llm_dict = load_checkpoint_config_dicts(draft_dir)
dflash_config = llm_dict.get("dflash_config", {}) or {}
contract = resolve_dflash_contract(root_dict, llm_dict)
# Domino applies its own label shift inside the correction head.
shift_label = contract.shift_label and not contract.is_domino
# Parse quantization config from the draft checkpoint directory.
# For FP16 draft checkpoints this returns QuantConfig() (no quant).
quant = _parse_quant(draft_dir, llm_dict)
if (quantize_w4a16_activations and _DEFAULT_QUANTIZE_ACTIVATIONS
and quant.quant_type == QUANT_NVFP4_A16):
# The checkpoint's input scales decide: modules without one fall back
# to W4A16, whatever the declared algorithm says.
quant = _weight_only_nvfp4(draft_dir,
replace(quant, quant_type=QUANT_NVFP4), "")
# A dense fc keeps the FP16 + FP32 projection (the draft model asserts
# this); a quantized fc runs at the checkpoint's precision, see ``model.py``.
if "fc" not in quant.excluded:
quant.excluded = list(quant.excluded) + ["fc"]
model_type = llm_dict.get("model_type", "qwen3")
_check_num_attention_heads(llm_dict["num_attention_heads"])
raw_layer_types = _parse_raw_layer_types(llm_dict)
layer_types = _parse_layer_types(llm_dict)
attention_layer_types = _parse_attention_layer_types(
llm_dict, llm_dict["num_hidden_layers"], model_type)
if (not attention_layer_types and raw_layer_types
and all(layer_type in _VALID_ATTENTION_LAYER_TYPES
for layer_type in raw_layer_types)):
attention_layer_types = list(raw_layer_types)
full_attention_only = (_is_gemma4_model_type(model_type)
and attention_layer_types
and all(layer_type == "full_attention"
for layer_type in attention_layer_types))
head_dim = llm_dict.get(
"global_head_dim" if full_attention_only else "head_dim",
llm_dict.get(
"head_dim",
llm_dict["hidden_size"] // llm_dict["num_attention_heads"]))
global_head_dim = int(llm_dict.get("global_head_dim", 0) or 0)
num_global_kv_heads = int(
llm_dict.get("num_global_key_value_heads", 0) or 0)
num_kv_heads = (
num_global_kv_heads if full_attention_only and num_global_kv_heads else
llm_dict.get("num_key_value_heads", llm_dict["num_attention_heads"]))
dual_rope_configs = _get_dual_rope_configs(llm_dict)
use_sw = llm_dict.get("use_sliding_window", False) or any(
layer_type == "sliding_attention" for layer_type in raw_layer_types)
sw_raw = llm_dict.get("sliding_window") if use_sw else None
sliding_window_size = int(sw_raw) if sw_raw is not None else -1
default_attention_scale_value = float(default_attention_scale(head_dim))
default_mask_token_id = 4 if _is_gemma4_model_type(model_type) else 248070
vocab_size = llm_dict.get("vocab_size", target_vocab_size)
if vocab_size is None:
raise ValueError("DFlash draft config must provide vocab_size")
draft_vocab_size = llm_dict.get("draft_vocab_size")
draft_vocab_size = int(vocab_size if draft_vocab_size is
None else draft_vocab_size)
return ModelConfig(
model_type=model_type,
hidden_size=llm_dict["hidden_size"],
num_hidden_layers=llm_dict["num_hidden_layers"],
num_attention_heads=llm_dict["num_attention_heads"],
num_key_value_heads=num_kv_heads,
intermediate_size=llm_dict["intermediate_size"],
head_dim=head_dim,
global_head_dim=global_head_dim,
num_global_key_value_heads=num_global_kv_heads,
rms_norm_eps=llm_dict.get("rms_norm_eps", 1e-6),
vocab_size=int(vocab_size),
rope_theta=_get_rope_theta(llm_dict),
max_position_embeddings=llm_dict.get("max_position_embeddings", 4096),
default_attention_scale=default_attention_scale_value,
rope_scaling=_select_rope_scaling(llm_dict),
partial_rotary_factor=_get_partial_rotary_factor(llm_dict),
hidden_activation=llm_dict.get("hidden_activation",
llm_dict.get("hidden_act", "silu")),
sliding_rope_config=dual_rope_configs.get("sliding_rope_config"),
full_rope_config=dual_rope_configs.get("full_rope_config"),
has_qk_norm=True,
has_value_norm=_get_has_value_norm(llm_dict, model_type),
attention_scaling=_get_attention_scaling(
llm_dict, head_dim, default_attention_scale_value),
attention_k_eq_v=bool(llm_dict.get("attention_k_eq_v", False)),
final_logit_softcapping=llm_dict.get("final_logit_softcapping", None),
torch_dtype=llm_dict.get("torch_dtype", "bfloat16"),
tie_word_embeddings=False,
sliding_window_size=sliding_window_size,
layer_types=layer_types,
attention_layer_types=attention_layer_types,
raw_layer_types=raw_layer_types,
rope_parameters=llm_dict.get("rope_parameters", None),
draft_vocab_size=draft_vocab_size,
reduced_vocab_size=(draft_vocab_size
if draft_vocab_size < int(vocab_size) else None),
is_dflash_draft_flag=True,
dflash_target_layer_ids=list(
dflash_config.get(
"target_layer_ids",
llm_dict.get("target_layer_ids", [1, 8, 15, 22, 29]))),
dflash_block_size=int(
dflash_config.get("block_size", llm_dict.get("block_size", 16))),
dflash_mask_token_id=int(
dflash_config.get(
"mask_token_id",
llm_dict.get("mask_token_id", default_mask_token_id))),
dflash_shift_label=shift_label,
quant=quant,
)
def make_dflash2_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float]) -> ModelConfig:
"""Build and validate the production DFlash2 draft contract."""
root_dict, llm_dict = load_checkpoint_config_dicts(draft_dir)
resolved = resolve_dflash_contract(root_dict, llm_dict)
if resolved.version != DFlashVersion.V2:
raise ValueError(
"DFlash2 draft checkpoint requires architecture DFlash2DraftModel")
config = make_dflash_draft_config(draft_dir, default_attention_scale)
if config.num_hidden_layers != 5:
raise ValueError(
"DFlash2 production checkpoint requires exactly five draft layers")
return replace(
config,
dflash_version=resolved.version,
dflash_target_layer_ids=list(resolved.target_layer_ids),
dflash_block_size=resolved.block_size,
dflash_mask_token_id=resolved.mask_token_id,
is_dflash_draft_flag=True,
dflash2_target_layer_ids=list(resolved.target_layer_ids),
dflash2_block_size=resolved.block_size,
dflash2_mask_token_id=resolved.mask_token_id,
dflash2_is_causal=resolved.is_causal,
dflash2_conv_kernel_size=resolved.conv_kernel_size,
dflash2_conv_group_size=resolved.conv_group_size,
dflash2_selector_rank=resolved.selector_rank,
dflash2_selector_top_k=resolved.selector_top_k,
dflash2_output_multiplier=resolved.output_multiplier,
dflash2_final_logit_softcapping=resolved.final_logit_softcapping,
)
def make_lilicorr_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float]) -> ModelConfig:
"""Build and validate the LiLiCorr draft contract (DFlash2 + lattice head)."""
root_dict, llm_dict = load_checkpoint_config_dicts(draft_dir)
resolved = resolve_dflash_contract(root_dict, llm_dict)
if not resolved.is_lilicorr:
raise ValueError("LiLiCorr draft checkpoint requires "
"dflash_config.projector_type='lilicorr'")
config = make_dflash2_draft_config(draft_dir, default_attention_scale)
if config.draft_vocab_size != config.vocab_size:
raise ValueError(
"LiLiCorr candidates require the full target vocabulary: the draft "
f"declares draft_vocab_size={config.draft_vocab_size}, "
f"vocab_size={config.vocab_size}")
head_hidden_size = resolved.lilicorr_hidden_size
if head_hidden_size == config.hidden_size:
raise ValueError(
"LiLiCorr heads as wide as the draft hidden size "
f"({config.hidden_size}) use an identity token_proj, which is not "
"supported; set dflash_config.lilicorr_hidden_size to the head width"
)
quant = config.quant
# LiLiCorr runs fc as the checkpoint's packed NVFP4 W4A16 GEMM. The draft
# ships no lm_head: an `*lm_head` exclusion glob becomes the exact name, which
# the base lm_head inheritance replaces when the base head is quantized.
excluded = [
name for name in quant.excluded
if name != "fc" and not fnmatch.fnmatchcase("lm_head", name)
]
if any(fnmatch.fnmatchcase("lm_head", name) for name in quant.excluded):
excluded.append("lm_head")
quant = replace(quant, excluded=excluded)
dflash_config = llm_dict.get("dflash_config") or {}
return replace(
config,
quant=quant,
attention_sink_bias=bool(
dflash_config.get("attention_sink_bias",
llm_dict.get("attention_sink_bias", False))),
dflash_projector_type="lilicorr",
lilicorr_candidate_topk=resolved.lilicorr_candidate_topk,
lilicorr_hidden_size=head_hidden_size,
lilicorr_num_layers=resolved.lilicorr_num_layers,
lilicorr_num_heads=resolved.lilicorr_num_heads,
lilicorr_mlp_ratio=resolved.lilicorr_mlp_ratio,
lilicorr_factor_dim=resolved.lilicorr_factor_dim,
lilicorr_vector_eps=resolved.lilicorr_vector_eps,
lilicorr_logit_scale=resolved.lilicorr_logit_scale,
)
def make_domino_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float]) -> ModelConfig:
"""Build and validate the Domino draft contract (DFlash V1 + correction head)."""
root_dict, llm_dict = load_checkpoint_config_dicts(draft_dir)
resolved = resolve_dflash_contract(root_dict, llm_dict)
if not resolved.is_domino:
raise ValueError(
"Domino draft checkpoint requires dflash_config.projector_type="
"'domino'")
config = make_dflash_draft_config(draft_dir, default_attention_scale)
# The in-engine pool projection is a split of the checkpoint's FP16/BF16
# embed_proj.0 and is filled by a load hook; keep it dense FP16.
if "domino_pool.z_projection" not in config.quant.excluded:
config.quant.excluded = list(
config.quant.excluded) + ["domino_pool.z_projection"]
return replace(
config,
dflash_version=resolved.version,
dflash_target_layer_ids=list(resolved.target_layer_ids),
dflash_block_size=resolved.block_size,
dflash_mask_token_id=resolved.mask_token_id,
is_dflash_draft_flag=True,
dflash_projector_type="domino",
domino_gru_hidden_dim=resolved.gru_hidden_dim,
domino_proj_dim=resolved.proj_dim,
domino_prefix_len=resolved.prefix_len,
domino_pool_size=min(resolved.pool_size, config.vocab_size),
domino_shift_label=resolved.shift_label,
)
def make_jetspec_draft_config(
draft_dir: str,
default_attention_scale: Callable[[int], float]) -> ModelConfig:
"""Build a JetSpec draft config from an official JetSpec checkpoint.
The public JetSpec Qwen3 checkpoint stores its metadata in ``dflash_config``
because JetSpec reuses the DFlash draft-head implementation. Persist the
exported runtime config under ``jetspec_config`` while also filling the
DFlash fields consumed by :class:`DFlashDraftModel`.
"""
_, llm_dict = load_checkpoint_config_dicts(draft_dir)
base = make_dflash_draft_config(draft_dir, default_attention_scale)
jetspec_config = (llm_dict.get("jetspec_config")
or llm_dict.get("dflash_config") or {})
target_layer_ids = list(
jetspec_config.get("target_layer_ids",
llm_dict.get("target_layer_ids", [])) or [])
if not target_layer_ids:
raise ValueError(
"JetSpec draft config requires target_layer_ids in config.json.")
block_size = int(
jetspec_config.get("block_size",
llm_dict.get("block_size", base.dflash_block_size)))
mask_token_id = int(
jetspec_config.get("mask_token_id",
llm_dict.get("mask_token_id", 151669)))
causal_head = bool(
jetspec_config.get("causal_head", llm_dict.get("causal_head", True)))
if not causal_head:
raise ValueError(
"JetSpec draft config requires causal_head=true; use DFlash for non-causal block drafts."
)
return replace(
base,
is_dflash_draft_flag=False,
is_jetspec_draft_flag=True,
jetspec_target_layer_ids=target_layer_ids,
jetspec_block_size=block_size,
jetspec_mask_token_id=mask_token_id,
jetspec_causal_head=causal_head,
dflash_target_layer_ids=target_layer_ids,
dflash_block_size=block_size,
dflash_mask_token_id=mask_token_id,
)
def _parse_num_deepstack_features(
config: dict,
model_type: str,
*,
root_config: Optional[Dict[str, Any]] = None,
) -> int:
"""Return the number of deepstack visual features for ONNX / runtime.
1. Explicit ``num_deepstack_features`` on the LLM (text) dict, then on
the root dict.
2. ``len(vision_config["deepstack_visual_indexes"])`` on the root config.
3. Fallback: ``3`` for ``qwen3_vl`` / ``qwen3_omni`` when still unknown, else ``0``.
"""
raw = config.get("num_deepstack_features")
if raw is not None:
return int(raw)
if root_config is not None:
raw_root = root_config.get("num_deepstack_features")
if raw_root is not None:
return int(raw_root)
vision = root_config.get("vision_config")
if isinstance(vision, dict):
indexes = vision.get("deepstack_visual_indexes")
if isinstance(indexes, (list, tuple)) and len(indexes) > 0:
return len(indexes)
root_mt = (root_config or {}).get("model_type") or ""
# Strict equality, not substring — ``"qwen3_omni" in "qwen3_omni_next"``
# is True and would otherwise report 3 deepstack features for Qwen3-Next Omni
# (whose ``deepstack_visual_indexes`` is empty -> 0 features).
if model_type in _DEEPSTACK_MODEL_TYPES or root_mt in _DEEPSTACK_MODEL_TYPES:
return 3
return 0
def _parse_accept_hidden_layer(
config: dict,
*,
root_config: Optional[Dict[str, Any]] = None,
) -> int:
"""Return ``accept_hidden_layer`` (Thinker decoder layer count whose output
is handed off to the Qwen3-Omni Talker), or -1 when not applicable.
Lookup order:
1. ``accept_hidden_layer`` at the LLM (text) dict top level. This is
the layout written by the standalone-Thinker quant export, which
promotes the field out of ``talker_config`` into the Thinker root.
2. ``root_config["talker_config"]["accept_hidden_layer"]`` for the
full multimodal HF config.
3. ``root_config["accept_hidden_layer"]`` as a defensive fallback for
standalone Talker checkpoints where this field already lives at
root.
"""
raw = config.get("accept_hidden_layer")
if raw is not None:
return int(raw)
if root_config is not None:
talker_cfg = root_config.get("talker_config")
if isinstance(talker_cfg, dict):
raw = talker_cfg.get("accept_hidden_layer")
if raw is not None:
return int(raw)
raw = root_config.get("accept_hidden_layer")
if raw is not None:
return int(raw)
return -1
def _validate_mtp_constraints(
*,
model_type: str,
mtp_num_hidden_layers: Optional[int],
mtp_use_dedicated_embeddings: bool,
) -> None:
"""Validate the currently supported MTP config subset."""
if mtp_num_hidden_layers is None and not mtp_use_dedicated_embeddings:
return
if (model_type or "").lower().startswith("nemotron_h"):
if mtp_use_dedicated_embeddings:
raise NotImplementedError(
"Dedicated MTP embeddings are not supported for Nemotron-H MTP."
)
return
if model_type not in _MTP_CONFIG_MODEL_TYPES:
raise NotImplementedError(
"MTP config parsing is only supported for Qwen3.5, dense Qwen3 "
"and Nemotron-H checkpoints.")
if mtp_num_hidden_layers != 1:
raise NotImplementedError(
"Only mtp_num_hidden_layers == 1 is supported for MTP.")
if mtp_use_dedicated_embeddings:
raise NotImplementedError(
"Dedicated MTP embeddings are not supported for MTP.")
# Nested ``eagle_config`` aux IDs name decoder-layer outputs; layer j's output
# is HF hidden-state index j + 1.
_EAGLE_CONFIG_LAYER_ID_OFFSET = 1
_EAGLE_DRAFT_FEATURE_KEYS = ("fc_norm", "norm_output",
"efficient_lm_head_config")
# Names used in errors where the checkpoint key is not self-explanatory.
_EAGLE_DRAFT_FEATURE_LABELS = {
"efficient_lm_head_config": "a low-rank candidate head"
}
# Checkpoint prefixes of the low-rank candidate head factors and the
# Eagle4DraftModel modules they load into.
_EAGLE_CANDIDATE_HEAD_KEY_PREFIXES = (
("efficient_lm_head_down_proj.", "candidate_head.down_proj."),
("efficient_lm_head.", "candidate_head.up_proj."),
)
# EAGLE drafts read the base embedding.
_EAGLE_SKIPPED_CHECKPOINT_KEYS = frozenset({"embed_tokens.weight"})
# Shapes supported by LowRankCandidateHeadPlugin.
_MAX_DRAFT_CANDIDATE_COUNT = 256
_MAX_DRAFT_CANDIDATE_RANK = 256
_DRAFT_CANDIDATE_RANK_MULTIPLE = 16
@dataclass(frozen=True)
class _EagleDraftFeatures:
fc_norm: bool = False
norm_output: bool = False
candidate_rank: int = 0
candidate_count: int = 0
def _eagle_draft_feature_keys(llm_dict: Dict[str, Any]) -> List[str]:
"""Return the EAGLE draft capability keys ``llm_dict`` turns on.
A key that is false, null or empty does not count.
"""
return [key for key in _EAGLE_DRAFT_FEATURE_KEYS if llm_dict.get(key)]
def _describe_eagle_draft_features(llm_dict: Dict[str, Any]) -> str:
"""Name the EAGLE draft capabilities ``llm_dict`` turns on ("" for none)."""
features = [
_EAGLE_DRAFT_FEATURE_LABELS.get(key, key)
for key in _eagle_draft_feature_keys(llm_dict)
]
if len(features) > 1:
return f"{', '.join(features[:-1])} and {features[-1]}"
return features[0] if features else ""
def _parse_eagle_target_layer_ids(
llm_dict: Dict[str, Any],
num_layers: Optional[int] = None) -> List[int]:
"""Resolve an EAGLE draft's aux hidden-state layers as HF indices.
Precedence: top-level ``eagle_aux_hidden_state_layer_ids`` (HF index,
as-is), then nested ``eagle_config.eagle_aux_hidden_state_layer_ids``
(layer outputs), then flat ``target_layer_ids`` (unchanged), else ``[]`` so
the base applies its default. ``num_layers`` is the base's decoder-layer
count, when known.
"""
top_level = llm_dict.get("eagle_aux_hidden_state_layer_ids")
if top_level:
return list(top_level)
eagle_config = llm_dict.get("eagle_config") or {}
if eagle_config.get("use_aux_hidden_state") is False:
raise ValueError(
"EAGLE draft sets eagle_config.use_aux_hidden_state=false; only "
"drafts that fuse base aux hidden states are supported.")
nested = eagle_config.get("eagle_aux_hidden_state_layer_ids")
if nested:
layer_outputs = [int(layer) for layer in nested]
if any(layer < 0 for layer in layer_outputs):
raise ValueError(
"eagle_config.eagle_aux_hidden_state_layer_ids must be "
f"non-negative, got {layer_outputs}.")
# TorchSpec concatenates aux states in list order, so sorting would
# permute the fc input columns.
if ("_torchspec_version" in llm_dict
and layer_outputs != sorted(layer_outputs)):
raise ValueError(
"TorchSpec eagle_config.eagle_aux_hidden_state_layer_ids "
f"must be ascending, got {layer_outputs}.")
layer_ids = sorted(layer + _EAGLE_CONFIG_LAYER_ID_OFFSET
for layer in layer_outputs)
# HF index num_layers is the final-norm output, not a layer output.
if num_layers is not None and any(layer >= num_layers
for layer in layer_ids):
raise ValueError(
f"EAGLE aux layer outputs {layer_outputs} need HF indices "
f"{layer_ids} below the base num_hidden_layers={num_layers}.")
return layer_ids
return list(llm_dict.get("target_layer_ids") or [])
def _parse_eagle_draft_features(
llm_dict: Dict[str, Any],
target_layer_ids: List[int]) -> _EagleDraftFeatures:
"""Parse and validate the optional EAGLE draft norms and candidate head."""
fc_norm = bool(llm_dict.get("fc_norm", False))
norm_output = bool(llm_dict.get("norm_output", False))
for key in ("norm_before_fc", "norm_before_residual"):
if llm_dict.get(key):
raise ValueError(
"EAGLE drafts with fc_norm, norm_output or a candidate head do "
f"not support {key}.")
if _is_gemma4_model_type(llm_dict.get("model_type", "")):
raise ValueError(
"EAGLE drafts with fc_norm, norm_output or a candidate head do not "
"support a Gemma4 draft layer.")
num_aux_hidden_states = llm_dict.get("num_aux_hidden_states")
if (num_aux_hidden_states is not None and target_layer_ids
and int(num_aux_hidden_states) != len(target_layer_ids)):
raise ValueError(
f"EAGLE draft num_aux_hidden_states={num_aux_hidden_states} "
f"does not match its {len(target_layer_ids)} aux layer IDs "
f"{target_layer_ids}.")
head = llm_dict.get("efficient_lm_head_config")
if not head:
return _EagleDraftFeatures(fc_norm=fc_norm, norm_output=norm_output)
head_type = head.get("type")
if head_type != "low_rank":
raise ValueError(
f"EAGLE candidate head type {head_type!r} is not supported; "
"expected 'low_rank'.")
vocab_size = int(llm_dict["vocab_size"])
count = int(head.get("num_candidates", 0) or 0)
max_count = min(vocab_size, _MAX_DRAFT_CANDIDATE_COUNT)
if not 0 < count <= max_count:
raise ValueError(
f"EAGLE candidate head num_candidates={count} must be in "
f"[1, {max_count}] (vocab_size={vocab_size}).")
rank = int(head.get("rank", 0) or 0)
if not 0 < rank <= _MAX_DRAFT_CANDIDATE_RANK:
raise ValueError(f"EAGLE candidate head rank={rank} must be in "
f"[1, {_MAX_DRAFT_CANDIDATE_RANK}].")
if rank % _DRAFT_CANDIDATE_RANK_MULTIPLE:
raise ValueError(
f"EAGLE candidate head rank={rank} must be a multiple of "
f"{_DRAFT_CANDIDATE_RANK_MULTIPLE}.")
# The engine's only hidden output is the final-norm output, which both the
# head and the next draft step read.
if not norm_output:
raise ValueError("An EAGLE candidate head requires norm_output=true.")
if llm_dict.get("final_logit_softcapping") is not None:
raise ValueError(
"An EAGLE candidate head does not support final_logit_softcapping."
)
draft_vocab_size = llm_dict.get("draft_vocab_size")
if draft_vocab_size is not None and int(draft_vocab_size) != vocab_size:
raise ValueError("An EAGLE candidate head scores the full vocabulary; "
f"draft_vocab_size={draft_vocab_size} must equal "
f"vocab_size={vocab_size}.")
return _EagleDraftFeatures(fc_norm=fc_norm,
norm_output=norm_output,
candidate_rank=rank,
candidate_count=count)
def require_dense_eagle_draft(llm_dict: Dict[str, Any], consumer: str) -> None:
"""Raise unless an EAGLE draft config describes the dense EAGLE-3 graph.
``fc_norm``, ``norm_output`` and low-rank candidate heads need an EAGLE-4
draft graph. ``consumer`` implements the dense graph alone and must not
silently treat such a draft as dense. A key that is false, null or empty
does not count.
"""
listed = _describe_eagle_draft_features(llm_dict)
if listed:
raise NotImplementedError(
f"{consumer} implements only the dense EAGLE3 draft and does not "
f"support {listed}. Export this draft with "
"tensorrt-edgellm-export instead.")
def _parse_raw_layer_types(config: dict) -> List[str]:
"""Return checkpoint layer type strings without canonicalization."""
raw = config.get("layers_block_type") or config.get("layer_types")
if raw is None:
return []
return [str(layer_type) for layer_type in raw]
def _is_nemotron_h_config(config: dict) -> bool:
return (config.get("model_type") or "").lower().startswith("nemotron_h")
def _canonical_layer_type(block_type: str, is_nemotron_h: bool) -> str:
"""Map one checkpoint block-type name onto a canonical layer label.
``"linear_attention"`` is ambiguous: it covers any sub-quadratic mixer, so
it denotes Mamba2 for NemotronH and GatedDeltaNet for Qwen3.5. The model
family therefore resolves it, along with ``"full_attention"``, which Gemma4
keeps verbatim for per-layer head_dim dispatch.
"""
bt = str(block_type).lower()
if bt == "linear_attention":
return LAYER_MAMBA if is_nemotron_h else LAYER_GDN
if "mamba" in bt:
return LAYER_MAMBA
if bt == "moe":
return LAYER_MOE
if "mlp" in bt:
return LAYER_MLP
if bt in _VALID_ATTENTION_LAYER_TYPES:
return LAYER_ATTN if is_nemotron_h else bt
return LAYER_ATTN
def _parse_mtp_layer_types(config: dict) -> List[str]:
"""Return the MTP draft stack's per-layer block types.
NemotronH declares the stack either as ``mtp_layers_block_type`` (a list,
which transformers >= 5.14 rewrites into the ``linear_attention`` /
``full_attention`` spelling) or as the legacy ``mtp_hybrid_override_pattern``
string (e.g. ``"*E"``).
"""
raw = config.get("mtp_layers_block_type")
if raw:
is_nemotron_h = _is_nemotron_h_config(config)
return [_canonical_layer_type(bt, is_nemotron_h) for bt in raw]
pattern = config.get("mtp_hybrid_override_pattern") or ""
return [
_HYBRID_PATTERN_MAP[ch] for ch in pattern if ch in _HYBRID_PATTERN_MAP
]
def _parse_layer_types(config: dict) -> List[str]:
"""Return per-layer block type list from config.
Reads ``layers_block_type`` or ``layer_types`` directly if present, each
entry resolved by :func:`_canonical_layer_type`. For models using
``hybrid_override_pattern`` (e.g. NemotronH), parses the pattern string
where ``M`` = mamba, ``-`` = mlp, ``*`` = attention. Falls back to all
attention layers.
"""
raw = config.get("layers_block_type") or config.get("layer_types")
if raw is not None:
is_nemotron_h = _is_nemotron_h_config(config)
return [_canonical_layer_type(bt, is_nemotron_h) for bt in raw]
pattern = config.get("hybrid_override_pattern")
if pattern is not None:
return [
_HYBRID_PATTERN_MAP[ch] for ch in pattern
if ch in _HYBRID_PATTERN_MAP
]
n = config["num_hidden_layers"]
return [LAYER_ATTN] * n
def _parse_puzzle_block_configs(config: dict,
layer_types: List[str]) -> List[dict]:
"""Keep the heterogeneous MoE geometry from Nemotron-H Puzzle."""
if config.get("model_type") != "nemotron_h_puzzle":
return []
blocks = config.get("block_configs")
if not isinstance(blocks, list) or len(blocks) != len(layer_types):
raise ValueError(
"Nemotron-H Puzzle requires one block_config per layer")
result = []
for index, (block, layer_type) in enumerate(zip(blocks, layer_types)):
if not isinstance(block, dict) or _canonical_layer_type(
block.get("block_type", ""), True) != layer_type:
raise ValueError(
f"Invalid Nemotron-H Puzzle block_config at layer {index}")
if layer_type == LAYER_MOE:
geometry = {
name: int(block.get(name, 0) or 0)
for name in ("n_routed_experts", "moe_intermediate_size",
"num_experts_per_tok")
}
if any(value <= 0 for value in geometry.values()):
raise ValueError(
f"Incomplete Nemotron-H Puzzle MoE geometry at layer {index}"
)
result.append(geometry)
else:
result.append({})
return result
def _parse_mamba_cfg(config: dict,
layer_types: List[str],
model_dir: str = "") -> Optional[MambaConfig]:
"""Return a MambaConfig if any layer is a mamba layer, else None.
``model_dir`` is used to detect ``conv_dim`` from the actual checkpoint
weight shape when the config does not declare it explicitly. This resolves
a circular dependency: ``conv_dim`` depends on ``n_groups`` and vice-versa.
"""
if LAYER_MAMBA not in layer_types:
return None
num_heads = config.get("mamba_num_heads", 0)
head_dim = config.get("mamba_head_dim", 0)
ssm_state_size = config.get("ssm_state_size", 0)
conv_kernel = config.get("conv_kernel", config.get("mamba_d_conv", 4))
d_inner = num_heads * head_dim
n_groups = config.get(
"n_groups",
config.get("mamba_n_groups", config.get("mamba_num_groups", 1)))
if "conv_dim" in config:
conv_dim = config["conv_dim"]
else:
# Try to read conv_dim from the actual checkpoint shape to avoid
# circular dependency with n_groups when neither key is in config.
detected = _detect_mamba_conv_dim(model_dir) if model_dir else 0
if detected > 0:
conv_dim = detected
else:
conv_dim = d_inner + 2 * n_groups * ssm_state_size
# Back-derive n_groups from conv_dim if not explicit in config
if n_groups == 1 and conv_dim > d_inner:
derived = (conv_dim - d_inner) // (2 * ssm_state_size)
n_groups = derived if derived > 0 else 1
return MambaConfig(
num_heads=num_heads,
head_dim=head_dim,
ssm_state_size=ssm_state_size,
conv_dim=conv_dim,
conv_kernel=conv_kernel,
n_groups=n_groups,
)
def _detect_mamba_conv_dim(model_dir: str) -> int:
"""Return conv_dim by reading the first Mamba conv1d.weight shape from the checkpoint.
The conv1d weight has shape ``[conv_dim, 1, conv_kernel]`` so ``shape[0]``
gives ``conv_dim`` directly — no tensor data is loaded. Returns 0 if
not found (caller falls back to formula-based derivation).
"""
try:
index_path = os.path.join(model_dir, "model.safetensors.index.json")
if os.path.exists(index_path):
with open(index_path) as f:
index = json.load(f)
weight_map: dict = index.get("weight_map", {})
shard_for_key: Optional[str] = None
target_key: Optional[str] = None
for k, shard in weight_map.items():
if k.endswith(".mixer.conv1d.weight"):
shard_for_key = shard
target_key = k
break
if shard_for_key and target_key:
shard_path = os.path.join(model_dir, shard_for_key)
from safetensors import safe_open
with safe_open(shard_path, framework="pt") as f:
return f.get_slice(target_key).get_shape()[0]
else:
single_path = os.path.join(model_dir, "model.safetensors")
if os.path.exists(single_path):
from safetensors import safe_open
with safe_open(single_path, framework="pt") as f:
for k in f.keys():
if k.endswith(".mixer.conv1d.weight"):
return f.get_slice(k).get_shape()[0]
except (OSError, KeyError, ImportError):
pass
return 0
def _parse_gdn_cfg(config: dict,
layer_types: List[str]) -> Optional[GdnConfig]:
"""Return a GdnConfig if any layer is a GDN (linear_attention) layer, else None."""
if LAYER_GDN not in layer_types:
return None
return GdnConfig(
num_key_heads=config.get("linear_num_key_heads", 0),
num_value_heads=config.get("linear_num_value_heads", 0),
key_head_dim=config.get("linear_key_head_dim", 0),
value_head_dim=config.get("linear_value_head_dim", 0),
conv_kernel=config.get("linear_conv_kernel_dim", 4),
output_gate_type=(config.get("output_gate_type")
or config.get("hidden_act", "silu")),
)
def _get_partial_rotary_factor(llm_dict: Dict[str, Any]) -> float:
"""Extract partial_rotary_factor from config dict.
Qwen3.5 stores this inside ``rope_parameters`` rather than at top level.
"""
prf = llm_dict.get("partial_rotary_factor")
if prf is not None:
return float(prf)
for key in ("rope_parameters", "rope_scaling"):
nested = llm_dict.get(key)
if not isinstance(nested, dict):
continue
if nested.get("partial_rotary_factor") is not None:
return float(nested["partial_rotary_factor"])
for attention_type in ("full_attention", "sliding_attention"):
attention_params = nested.get(attention_type)
if (isinstance(attention_params, dict) and
attention_params.get("partial_rotary_factor") is not None):
return float(attention_params["partial_rotary_factor"])
return 1.0
def _detect_has_qk_norm(model_dir: str, model_type: str = "") -> bool:
"""Detect QK-norm by scanning checkpoint key names.
``*.q_norm.weight`` (Qwen3 convention) is model-agnostic. The HunYuan V1
``*.query_layernorm.weight`` spelling is only honored for model families
whose loader remaps it onto ``q_norm`` (see ``_hunyuan_key_remap``);
other families using that key name for unrelated norms must not trip
the fused QK-norm path.
"""
keys = _checkpoint_weight_keys(model_dir)
if any(".q_norm.weight" in k for k in keys):
return True
return (model_type in _QUERY_LAYERNORM_KEY_MODEL_TYPES
and any(".query_layernorm.weight" in k for k in keys))
def _checkpoint_weight_keys(model_dir: str) -> List[str]:
"""Return checkpoint tensor keys, ignoring stale shard indexes when needed."""
index_path = os.path.join(model_dir, "model.safetensors.index.json")
single_path = os.path.join(model_dir, "model.safetensors")
if os.path.exists(index_path):
try:
with open(index_path) as f:
index = json.load(f)
weight_map = index.get("weight_map", {})
missing_shards = {
shard
for shard in set(weight_map.values())
if not os.path.exists(os.path.join(model_dir, shard))
}
if not missing_shards or not os.path.exists(single_path):
return list(weight_map.keys())
except (OSError, json.JSONDecodeError):
pass
if os.path.exists(single_path):
try:
from safetensors import safe_open
with safe_open(single_path, framework="pt") as f:
return list(f.keys())
except (OSError, ImportError):
pass
return []
_VL_LLM_PREFIXES = ("language_model.", "text_model.", "llm.", "thinker.")
def _strip_vl_prefix(name: str) -> str:
"""Strip a known VL wrapper prefix (e.g. ``language_model.``) if present."""
if name.startswith("model.language_model."):
return "model." + name[len("model.language_model."):]
if name.startswith("model.visual."):
return "visual." + name[len("model.visual."):]
for prefix in _VL_LLM_PREFIXES:
if name.startswith(prefix):
return name[len(prefix):]
return name
def _normalize_module_name(name: str) -> str:
"""Normalise a checkpoint / hf_quant_config module name to the short
namespace that ``make_linear`` uses (``layers.N...``, ``lm_head``, etc.).
LLM ``make_linear`` callers pass names without any ``model.`` prefix
(see ``modeling_default.py``: ``module_name=f"layers.{i}.mlp.gate_proj"``).
For multimodal checkpoints whose keys are
``model.language_model.layers.N...`` the entire compound prefix must be
stripped so the resulting short name matches what ``module_quant_type``
looks up. Single VL prefixes and bare ``model.`` follow the same rule.
Used by ``_effective_excluded_modules``, ``_detect_modelopt_unquantized_linears``,
and ``_parse_mixed_precision`` so that ``excluded``, ``layer_overrides``,
and ``module_name`` all share the same name space.
"""
if name.startswith("model.language_model."):
return name[len("model.language_model."):]
if name.startswith("thinker.model."):
return name[len("thinker.model."):]
# Talker sub-LLM keys share the thinker short-name space after
# ``_make_sub_model_dir(key_prefix='talker.')`` staging.
if name.startswith("talker.model."):
return name[len("talker.model."):]
# Bare submodel prefixes (consolidated Qwen3-Omni roots): the visual /
# audio / code_predictor builders use short names without them.
if name.startswith("thinker."):
return name[len("thinker."):]
if name.startswith("talker."):
return name[len("talker."):]
if name.startswith("model.decoder."):
return name[len("model.decoder."):]
for prefix in _VL_LLM_PREFIXES + ("model.", ):
if name.startswith(prefix):
return name[len(prefix):]
return name
def _detect_unquantized_modules(model_dir: str) -> List[str]:
"""Return module names whose weights are plain float (not int4 quantized).
Some layers (often ``lm_head``) use ``*.weight`` instead of ``*.qweight``.
Checkpoint wrapper prefixes (``model.``, ``language_model.``, etc.) are
stripped so that the returned names match the short names used by
``make_linear()``.
"""
all_keys = _checkpoint_weight_keys(model_dir)
# Top-level module prefix = everything before the last dot segment
qweight_modules = {
k.rsplit(".", 1)[0]
for k in all_keys if k.endswith(".qweight")
}
weight_modules = {
k.rsplit(".", 1)[0]
for k in all_keys if k.endswith(".weight")
}
excluded: List[str] = [
_normalize_module_name(m) for m in (weight_modules - qweight_modules)
]
# lm_head may have neither .weight nor .qweight when tie_word_embeddings=True
# (the checkpoint omits lm_head.weight entirely). Treat it as FP16 so that
# tie_weights() can clone embed_tokens.weight into it after loading.
all_linear_stripped = {
_normalize_module_name(m)
for m in qweight_modules | weight_modules
}
if "lm_head" not in all_linear_stripped:
excluded.append("lm_head")
return _with_gdn_fused_exclusions(excluded)
_GDN_INPUT_PROJ_MODULES = ("in_proj_qkv", "in_proj_z", "in_proj_b",
"in_proj_a")
# Fused HF GDN projection names (as they appear in a ModelOpt ``ignore`` /
# ``modules_to_not_convert`` list) mapped to the split projections
# trt-edge-llm actually builds. The checkpoint loader splits
# ``in_proj_qkvz`` -> ``in_proj_qkv`` / ``in_proj_z`` and ``in_proj_ba`` ->
# ``in_proj_b`` / ``in_proj_a`` (Qwen3-Next family). Excluding only the fused
# name would otherwise leave the split Linears at the dominant quant type
# (e.g. NVFP4) even though their weights are plain FP16, so ``make_linear``
# would build an NVFP4Linear against an unquantized weight.
_GDN_FUSED_PROJ_SPLITS: Dict[str, Tuple[str, ...]] = {
"in_proj_qkvz": ("in_proj_qkv", "in_proj_z"),
"in_proj_ba": ("in_proj_b", "in_proj_a"),
}
def _with_gdn_fused_exclusions(modules: List[str]) -> List[str]:
"""Reconcile GDN input-projection exclusions with the split/fused forms.
Expands fused HF names (``in_proj_qkvz`` / ``in_proj_ba``) present in the
ignore list into the split projections the model builds, and adds the
synthetic ``in_proj_fused`` when all four split projections are FP16.
"""
result = set(modules)
# Fused HF name -> split projections (so an excluded ``in_proj_qkvz``
# also excludes the ``in_proj_qkv`` / ``in_proj_z`` the model builds).
for module in list(result):
for fused, splits in _GDN_FUSED_PROJ_SPLITS.items():
suffix = f".{fused}"
if module.endswith(suffix):
prefix = module[:-len(suffix)]
for split in splits:
result.add(f"{prefix}.{split}")
break
by_prefix: Dict[str, set] = {}
for module in result:
for proj in _GDN_INPUT_PROJ_MODULES:
suffix = f".{proj}"
if module.endswith(suffix):
by_prefix.setdefault(module[:-len(suffix)], set()).add(proj)
break
for prefix, projs in by_prefix.items():
if all(proj in projs for proj in _GDN_INPUT_PROJ_MODULES):
result.add(f"{prefix}.in_proj_fused")
return sorted(result)
def _detect_quantized_modules(model_dir: str) -> List[str]:
"""Return modules that have checkpoint quantization sidecars."""
all_keys = _checkpoint_weight_keys(model_dir)
suffixes = (".qweight", ".weight_scale", ".weight_scale_2", ".input_scale",
".scales")
modules = {
_strip_vl_prefix(k.rsplit(".", 1)[0])
for k in all_keys if k.endswith(suffixes)
}
return sorted(modules)
def _effective_excluded_modules(model_dir: str,
excluded: List[str]) -> List[str]:
"""Drop exclusions contradicted by quantized tensors in the checkpoint,
and normalize remaining names so they match what ``make_linear`` looks up.
Normalisation mirrors ``_parse_mixed_precision``'s strip on
``layer_overrides`` keys (``language_model.`` / ``text_model.`` / ``llm.``
/ ``model.``) so a checkpoint that writes ``model.visual.blocks.X.Y`` to
``exclude_modules`` matches the ``visual.blocks.X.Y`` ``module_name`` the
modeling code passes.
"""
quantized_modules = set(_detect_quantized_modules(model_dir))
normalized = [
_normalize_module_name(module) for module in excluded
if _strip_vl_prefix(module) not in quantized_modules
]
return _with_gdn_fused_exclusions(normalized)
def _detect_modelopt_unquantized_linears(model_dir: str,
submodel_prefix: str = ""
) -> List[str]:
"""Return module_name strings of Linears the ModelOpt checkpoint left unquantized.
A ModelOpt-quantized Linear stores both ``<name>.weight`` (packed) and
``<name>.weight_scale`` (scale tensor; FP8 / NVFP4 / MXFP8 / AWQ / INT8-SQ
all emit this). Linears that ModelOpt skipped — typically because their
wildcard had ``enable: False`` (visual / audio / lm_head) — only have
``<name>.weight``.
We compute the set of "has .weight without .weight_scale" modules from the
checkpoint index, then return them in the short form ``make_linear`` uses
(leading ``model.`` and VL wrapper prefixes stripped). Norm and embedding
names also fall into this set but are harmless: ``make_linear`` only
consults ``excluded`` for paths that actually go through it (i.e. real
Linears), so extra entries are inert.
Used by ``_parse_quant`` to plug a long-standing gap: for dominant-quant
checkpoints (``quant_algo: FP8 / NVFP4 / W4A16_AWQ / ...``) ModelOpt does
NOT populate ``exclude_modules`` even when whole submodules (visual tower,
audio encoder) were skipped during PTQ. Without this augmentation,
``make_linear``'s dominant fallback would build NVFP4Linear / FP8Linear
against FP16 weights → shape mismatch on export.
Complements ``_effective_excluded_modules`` (which drops false-positive
excludes when the checkpoint contradicts them). This helper supplies the
opposite direction: false-negative excludes when ``exclude_modules`` is
silent on a submodule that was actually skipped during PTQ.
"""
all_keys = _checkpoint_weight_keys(model_dir)
if not all_keys:
return []
# Omni roots hold every sub-model's tensors. Restrict the scan, or another
# sub-model's unquantized Linears would be normalised onto this one's
# module names and silently unquantize them.
if submodel_prefix:
all_keys = [k for k in all_keys if k.startswith(submodel_prefix)]
weight_modules = {
k.rsplit(".", 1)[0]
for k in all_keys if k.endswith(".weight")
}
scale_modules = {
k.rsplit(".", 1)[0]
for k in all_keys if k.endswith(".weight_scale")
}
unquantized = weight_modules - scale_modules
excluded = set()
for name in unquantized:
normalized = _normalize_module_name(name)
if normalized.endswith(".self_attn.qkv_proj"):
prefix = normalized[:-len("qkv_proj")]
excluded.update(f"{prefix}{proj}"
for proj in ("q_proj", "k_proj", "v_proj"))
elif normalized.endswith(".mlp.gate_up_proj"):
prefix = normalized[:-len("gate_up_proj")]
excluded.update(f"{prefix}{proj}"
for proj in ("gate_proj", "up_proj"))
else:
excluded.add(normalized)
# Normalise to the same name space ``layer_overrides`` and ``excluded``
# use, so ``make_linear`` finds entries via its ``module_name`` lookup.
return _with_gdn_fused_exclusions(sorted(excluded))
def _scope_exclusions(patterns: List[str], submodel_prefix: str) -> List[str]:
"""Keep only exclusions that apply to *submodel_prefix*, prefix stripped.
Omni roots share one ignore list across thinker/talker, and short-name
normalisation later drops both prefixes. A talker-scoped glob such as
``talker.model.layers.0.mlp.shared_expert*`` would therefore also unquantize
the thinker's own shared_expert. A leading ``*`` marks a submodel-agnostic
glob and is kept as-is.
"""
if not submodel_prefix:
return patterns
scoped = []
for pat in patterns:
if pat.startswith(submodel_prefix):
scoped.append(pat[len(submodel_prefix):])
elif pat.startswith("*") or pat == "":
scoped.append(pat)
return scoped
def _grouped_quant_type(group: dict, root_format: str,
quant_method: str) -> Optional[str]:
group_format = str(group.get("format", root_format)).lower()
weights = group.get("weights") or {}
inputs = group.get("input_activations") or {}
# NVFP4 weights without quantized activations execute as W4A16.
if "nvfp4" in group_format:
return QUANT_NVFP4 if inputs else QUANT_NVFP4_A16
if (str(weights.get("type", "")).lower() == "float"
and int(weights.get("num_bits", 0) or 0) == 4 and not inputs):
return QUANT_NVFP4_A16
# ModelOpt writes weight-only int4 groups only for W4A16_AWQ (uint8
# packed); compressed-tensors int4 uses another packing.
if (quant_method == "modelopt" and not inputs
and str(weights.get("type", "")).lower() == "int"
and int(weights.get("num_bits", 0) or 0) == 4):
return QUANT_INT4_AWQ_MODELOPT
if (str(weights.get("type", "")).lower() == "float"
and str(inputs.get("type", "")).lower() == "float"):
weight_bits = int(weights.get("num_bits", 0) or 0)
input_bits = int(inputs.get("num_bits", 0) or 0)
if weight_bits == 4 and input_bits == 4:
return QUANT_NVFP4
if weight_bits == 8 and input_bits == 8:
return QUANT_FP8
return None
def _parse_grouped_nvfp4(model_dir: str,
config: dict,
submodel_prefix: str = "") -> Optional[QuantConfig]:
qc = config.get("quantization_config")
if not isinstance(qc, dict):
return None
quant_method = str(qc.get("quant_method", "")).lower()
if quant_method not in ("compressed-tensors", "modelopt"):
return None
config_groups = qc.get("config_groups") or {}
root_format = str(qc.get("format", ""))
groups = [(group, _grouped_quant_type(group, root_format, quant_method))
for group in config_groups.values()]
if not any(quant_type in (QUANT_NVFP4, QUANT_NVFP4_A16)
for _, quant_type in groups):
return None
# Groups naming another algorithm (e.g. Q8_0, IQ2_XXS) are described by
# the per-layer metadata, which the full parsers read.
if any(quant_type is None and group.get("quant_algo")
for group, quant_type in groups):
return None
quantized_layers = {}
global_types = []
for group, quant_type in groups:
if quant_type is None:
continue
group_size = int((group.get("weights") or {}).get(
"group_size",
16 if quant_type in (QUANT_NVFP4, QUANT_NVFP4_A16) else 1))
# _parse_mixed_precision re-reads quant_algo as a ModelOpt algo name.
# A grouped FP8 weight is block-scaled.
algo = quant_type
if quant_type == QUANT_FP8 and group_size > 1:
algo = "FP8_BLOCK_SCALES"
elif quant_type == QUANT_INT4_AWQ_MODELOPT:
algo = "W4A16_AWQ"
for target in group.get("targets") or ():
if target == "Linear":
global_types.append((quant_type, group_size))
else:
quantized_layers[target] = {
"quant_algo": algo,
"group_size": group_size,
}
excluded = _effective_excluded_modules(
model_dir,
_scope_exclusions(list(qc.get("ignore", [])), submodel_prefix))
if quantized_layers:
num_experts, top_k = _moe_routing(config)
dominant, group_size, overrides, layer_group_sizes = (
_parse_mixed_precision(quantized_layers,
num_experts=num_experts,
top_k=top_k))
# Named groups can retain targets left unquantized in the checkpoint.
# Reconcile only those overrides with their serialized weight payloads.
excluded.extend(name for name in _detect_modelopt_unquantized_linears(
model_dir, submodel_prefix)
if name in overrides and name not in excluded)
return QuantConfig(
quant_type=dominant,
group_size=group_size,
kv_cache_quant=_compressed_kv_cache_quant(qc),
excluded=excluded,
layer_overrides=overrides,
layer_group_sizes=layer_group_sizes,
is_mixed_precision=True,
)
quant_type, group_size = (global_types[0] if global_types else
(QUANT_NVFP4, 16))
quant_type = _select_nvfp4_quant_type(quant_type)
excluded.extend(module for module in _detect_modelopt_unquantized_linears(
model_dir, submodel_prefix) if module not in excluded)
return QuantConfig(
quant_type=quant_type,
group_size=group_size,
kv_cache_quant=_compressed_kv_cache_quant(qc),
excluded=excluded,
)
def _parse_compressed_integer(
model_dir: str,
config: dict,
submodel_prefix: str = "") -> Optional[QuantConfig]:
"""Lower uniform compressed-tensors integer weights to A16 execution."""
qc = config.get("quantization_config")
if not isinstance(qc, dict) or str(qc.get(
"quant_method",
"")).lower() not in ("compressed-tensors", "compressed_tensors"):
return None
contracts = []
for group in (qc.get("config_groups") or {}).values():
weights = group.get("weights") or {}
if str(weights.get("type", "")).lower() != "int":
return None
# Activation observers do not change the serialized integer weight
# layout. Edge-LLM consumes the same weights through W4A16/W8A16 when
# the checkpoint's W4A4/W8A8 execution contract is unavailable.
bits = int(weights.get("num_bits") or 0)
if bits not in (4, 8):
raise ValueError(
f"unsupported compressed-tensors integer width {bits}; "
"expected 4 or 8")
quant_type = QUANT_INT4_GPTQ if bits == 4 else QUANT_INT8_GPTQ
group_size = int(
weights.get("group_size") or (128 if bits == 4 else 0))
contracts.append((quant_type, group_size))
if not contracts:
raise ValueError(
"unsupported compressed-tensors checkpoint: config_groups is empty"
)
if len(set(contracts)) != 1:
raise ValueError(
"mixed compressed-tensors integer layouts are not supported")
quant_type, group_size = contracts[0]
excluded = _effective_excluded_modules(
model_dir,
_scope_exclusions(list(qc.get("ignore", [])), submodel_prefix))
excluded.extend(module for module in _detect_modelopt_unquantized_linears(
model_dir, submodel_prefix) if module not in excluded)
return QuantConfig(
quant_type=quant_type,
group_size=group_size,
gptq_zero_point_offset=_compressed_zero_point_offset(
model_dir,
qc.get("config_groups") or {}),
kv_cache_quant=_compressed_kv_cache_quant(qc),
excluded=excluded,
)
def _compressed_zero_point_offset(model_dir: str, config_groups: dict) -> int:
"""Return the packed-integer zero-point convention declared by the checkpoint."""
integer_weights = [
group.get("weights") or {} for group in config_groups.values()
if str((group.get("weights") or {}).get("type", "")).lower() == "int"
]
if not integer_weights:
return 1
if any(weights.get("symmetric") is False for weights in integer_weights):
keys = _checkpoint_weight_keys(model_dir)
if not any(key.endswith(".weight_zero_point") for key in keys):
raise ValueError(
"asymmetric compressed-tensors weights require explicit "
"weight_zero_point tensors")
# compressed-tensors stores actual zero points; it never uses GPTQ's
# historical zero-minus-one convention.
return 0
def _compressed_kv_cache_quant(qc: dict) -> Optional[str]:
"""Parse the compressed-tensors KV-cache quantization contract."""
scheme = qc.get("kv_cache_scheme")
if not scheme:
return None
if not isinstance(scheme, dict):
raise ValueError(
"compressed-tensors kv_cache_scheme must be an object")
value_type = str(scheme.get("type", "")).lower()
bits = int(scheme.get("num_bits") or 0)
if value_type == "float" and bits == 8:
return "fp8"
raise ValueError("unsupported compressed-tensors KV-cache quantization: "
f"type={value_type!r}, bits={bits}")
def _parse_quant(model_dir: str,
config: dict,
submodel_prefix: str = "") -> QuantConfig:
"""Determine quantisation config from the checkpoint metadata and tensors.
NVFP4 modules stored without a calibrated activation scale execute as
W4A16, whatever the metadata declares.
"""
return _weight_only_nvfp4(
model_dir, _parse_declared_quant(model_dir, config, submodel_prefix),
submodel_prefix)
def _weight_only_nvfp4(model_dir: str, quant: QuantConfig,
submodel_prefix: str) -> QuantConfig:
"""Demote NVFP4 modules without an input scale in the checkpoint to W4A16."""
if QUANT_NVFP4 not in (quant.quant_type, *quant.layer_overrides.values()):
return quant
keys = set(_checkpoint_weight_keys(model_dir))
calibrated, weight_only = set(), set()
for suffix, input_suffix in ((".weight_scale_2", ".input_scale"),
(".weight_global_scale",
".input_global_scale")):
for key in keys:
if not key.endswith(suffix):
continue
prefix = key[:-len(suffix)]
name = _normalize_module_name(
prefix[len(submodel_prefix):] if submodel_prefix
and prefix.startswith(submodel_prefix) else prefix)
(calibrated if prefix +
input_suffix in keys else weight_only).add(name)
if not weight_only:
return quant
overrides = dict(quant.layer_overrides)
fallback = QUANT_FP16 if quant.is_mixed_precision else quant.quant_type
for name in weight_only:
if overrides.get(name, fallback) == QUANT_NVFP4:
overrides[name] = QUANT_NVFP4_A16
quant_type = quant.quant_type
if quant_type == QUANT_NVFP4 and not calibrated:
quant_type = QUANT_NVFP4_A16
overrides = {
name: (QUANT_NVFP4_A16 if value == QUANT_NVFP4 else value)
for name, value in overrides.items()
}
return replace(quant, quant_type=quant_type, layer_overrides=overrides)
def _parse_declared_quant(model_dir: str,
config: dict,
submodel_prefix: str = "") -> QuantConfig:
"""Determine quantisation config from hf_quant_config.json or config.json.
Checkpoint-provided W4A16 ``lm_head`` (``W4A16_NVFP4`` in
``quantized_layers``, e.g. Qwen3.6-35B-A3B-NVFP4) maps to
:data:`QUANT_NVFP4_A16`. Excluded FP16/BF16 heads stay FP16; export does
not invent packed NVFP4 for them.
"""
compressed_integer = _parse_compressed_integer(model_dir, config,
submodel_prefix)
if compressed_integer is not None:
return compressed_integer
grouped_nvfp4 = _parse_grouped_nvfp4(model_dir, config, submodel_prefix)
if grouped_nvfp4 is not None:
return grouped_nvfp4
# ---- Sidecar hf_quant_config.json ---------------------------------------
hf_path = os.path.join(model_dir, "hf_quant_config.json")
if os.path.exists(hf_path):
with open(hf_path) as f:
hq = json.load(f)
q = hq.get("quantization", {})
algo = (q.get("quant_algo") or "").upper()
if "AWQ" in algo and "W4A16" in algo:
# Drop exclusions that the checkpoint contradicts (false positives),
# then augment with submodules ModelOpt actually left unquantized
# (false negatives — typically visual tower / audio encoder).
excluded = _effective_excluded_modules(
model_dir,
_scope_exclusions(list(q.get("exclude_modules", [])),
submodel_prefix))
excluded.extend(m for m in _detect_modelopt_unquantized_linears(
model_dir, submodel_prefix) if m not in excluded)
return QuantConfig(
quant_type=QUANT_INT4_AWQ_MODELOPT,
group_size=int(q.get("group_size", 128)),
excluded=excluded,
)
if algo == "MIXED_PRECISION":
quantized_layers = q.get("quantized_layers", {})
dominant, group_size, layer_overrides, layer_group_sizes = _parse_mixed_precision(
quantized_layers,
config.get("model_type") or "", *_moe_routing(config))
return QuantConfig(
quant_type=dominant,
group_size=group_size,
kv_cache_quant=_detect_llm_kv_cache_fp8(model_dir),
visual_mha_quant=_detect_visual_mha_fp8(model_dir),
excluded=_effective_excluded_modules(
model_dir,
_scope_exclusions(list(q.get("exclude_modules", [])),
submodel_prefix)),
layer_overrides=layer_overrides,
layer_group_sizes=layer_group_sizes,
is_mixed_precision=True,
)
qt = _select_nvfp4_quant_type(_algo_to_quant_type(algo))
gs = int(q.get("group_size", 1))
if qt == QUANT_MXFP8 and gs == 1:
gs = 32 # MXFP8 default block_size
excluded = _effective_excluded_modules(
model_dir,
_scope_exclusions(list(q.get("exclude_modules", [])),
submodel_prefix))
excluded.extend(m for m in _detect_modelopt_unquantized_linears(
model_dir, submodel_prefix) if m not in excluded)
return QuantConfig(
quant_type=qt,
group_size=gs,
kv_cache_quant=_detect_llm_kv_cache_fp8(model_dir),
visual_mha_quant=_detect_visual_mha_fp8(model_dir),
excluded=excluded,
)
# ---- Embedded quantization_config in config.json ------------------------
qc = config.get("quantization_config")
if qc is None:
return QuantConfig()
# Embedded block with ``quant_algo`` (export tool formats)
if "quant_algo" in qc:
algo = (qc.get("quant_algo") or "").upper()
if algo == "MIXED_PRECISION":
dominant, group_size, layer_overrides, layer_group_sizes = _parse_mixed_precision(
qc.get("quantized_layers", {}),
config.get("model_type") or "", *_moe_routing(config))
return QuantConfig(
quant_type=dominant,
group_size=group_size,
kv_cache_quant=_detect_llm_kv_cache_fp8(model_dir),
visual_mha_quant=_detect_visual_mha_fp8(model_dir),
excluded=_effective_excluded_modules(
model_dir,
_scope_exclusions(
list(qc.get("exclude_modules", qc.get("ignore", []))),
submodel_prefix)),
layer_overrides=layer_overrides,
layer_group_sizes=layer_group_sizes,
is_mixed_precision=True,
)
if "W4A16" in algo and "AWQ" in algo:
return QuantConfig(
quant_type=QUANT_INT4_AWQ_MODELOPT,
group_size=int(qc.get("group_size", 128)),
excluded=_effective_excluded_modules(
model_dir,
_scope_exclusions(list(qc.get("ignore", [])),
submodel_prefix)),
)
group_size = 1
cg = qc.get("config_groups", {})
if cg:
first_group = next(iter(cg.values()), {})
group_size = int(
first_group.get("weights", {}).get("group_size", 1))
elif qc.get("group_size") is not None:
group_size = int(qc.get("group_size"))
quant_type = _select_nvfp4_quant_type(_algo_to_quant_type(algo))
# NVFP4 uses a fixed FP8-block group size of 16. Minimal ModelOpt
# ``quantization_config`` blocks (``quant_algo`` only, no
# ``config_groups`` / ``group_size`` — e.g. Qwen3-Omni Next NVFP4)
# omit it, so default it here rather than leaving the per-tensor 1.
if quant_type in (QUANT_NVFP4, QUANT_NVFP4_A16) and group_size == 1:
group_size = 16
kv = qc.get("kv_cache_scheme")
kv_str = "fp8" if kv else None
return QuantConfig(
quant_type=quant_type,
group_size=group_size,
kv_cache_quant=kv_str,
excluded=_effective_excluded_modules(
model_dir,
_scope_exclusions(list(qc.get("ignore", [])),
submodel_prefix)),
)
# quant_method == awq (column-packed int4 checkpoints)
if qc.get("quant_method") == "awq":
return QuantConfig(
quant_type=QUANT_INT4_AWQ,
group_size=int(qc.get("group_size", 128)),
excluded=_detect_unquantized_modules(model_dir),
)
# quant_method == gptq
if qc.get("quant_method") == "gptq":
return QuantConfig(
quant_type=QUANT_INT4_GPTQ,
group_size=int(qc.get("group_size", 128)),
gptq_zero_point_offset=_detect_gptq_zero_point_offset(
model_dir, qc),
excluded=_detect_unquantized_modules(model_dir),
)
return QuantConfig()
def _detect_gptq_zero_point_offset(model_dir: str, qc: dict) -> int:
"""Infer whether GPTQ qzeros stores ``zero`` or ``zero - 1``.
Older symmetric GPTQ checkpoints used by Qwen3 store packed ``0x77777777``
for a real zero point of 8. Qwen3.5 stores packed ``0x88888888`` for the
same real zero point. Inspecting a tiny qzeros sample lets both variants
share the same repacking path without depending on GPTQModel/Optimum.
"""
if not bool(qc.get("sym", False)):
return 1
try:
from safetensors import safe_open
index_path = os.path.join(model_dir, "model.safetensors.index.json")
if os.path.exists(index_path):
with open(index_path) as f:
weight_map: dict = json.load(f).get("weight_map", {})
qzeros_key = next((k for k in weight_map if k.endswith(".qzeros")),
None)
if qzeros_key is None:
return 1
shard_path = os.path.join(model_dir, weight_map[qzeros_key])
else:
shard_path = os.path.join(model_dir, "model.safetensors")
if not os.path.exists(shard_path):
return 1
with safe_open(shard_path, framework="pt") as f:
qzeros_key = next(
(k for k in f.keys() if k.endswith(".qzeros")), None)
if qzeros_key is None:
return 1
with safe_open(shard_path, framework="pt") as f:
qzeros = f.get_tensor(qzeros_key).flatten()[:1024].cpu().tolist()
except Exception:
return 1
if not qzeros:
return 1
nibbles = []
for value in qzeros:
packed = int(value) & 0xFFFFFFFF
nibbles.extend((packed >> (4 * i)) & 0xF for i in range(8))
if nibbles and all(v == 8 for v in nibbles):
return 0
return 1
def _algo_to_quant_type(algo: str) -> str:
algo = algo.upper()
if algo == "IQ2_XXS":
return QUANT_IQ2_XXS
if algo == "Q8_0":
return QUANT_Q8_0
# MXFP8 per-block must be checked before generic FP8
if "FP8_PB" in algo or "MXFP8" in algo:
return QUANT_MXFP8
if "FP8" in algo:
return QUANT_FP8
if "FP4" in algo or "NVFP4" in algo:
# Weight-only NVFP4 (ModelOpt ``W4A16_NVFP4``) keeps FP16 activations.
return QUANT_NVFP4_A16 if "A16" in algo else QUANT_NVFP4
# W4A16_AWQ from ModelOpt unified checkpoints uses prepacked uint8 weights;
# plain AWQ / INT4_AWQ from HuggingFace uses column-packed int32 qweight.
if "W4A16" in algo and "AWQ" in algo:
return QUANT_INT4_AWQ_MODELOPT
if "AWQ" in algo or "INT4_AWQ" in algo:
return QUANT_INT4_AWQ
if "W8A8" in algo or "INT8" in algo:
return QUANT_INT8_SQ
return QUANT_FP16
def _select_nvfp4_quant_type(quant_type: str) -> str:
"""Resolve packed NVFP4 weights to the requested activation precision."""
if quant_type == QUANT_NVFP4 and not _DEFAULT_QUANTIZE_ACTIVATIONS:
return QUANT_NVFP4_A16
return quant_type
# Fp16MoePlugin routes only these expert counts and at most this top-k.
_FP16_MOE_EXPERT_COUNTS = (128, 256)
_FP16_MOE_MAX_TOP_K = 8
_MOE_EXPERT_COUNT_KEYS = ("num_experts", "num_local_experts",
"n_routed_experts", "moe_num_experts")
def _fp16_moe_routes(num_experts: int, top_k: int) -> bool:
return (num_experts in _FP16_MOE_EXPERT_COUNTS
and 0 < top_k <= _FP16_MOE_MAX_TOP_K)
def _moe_routing(config: dict) -> "tuple[int, int]":
"""``(num_experts, top_k)`` of an HF config dict, zeros for a dense model."""
count = next((int(config[key])
for key in _MOE_EXPERT_COUNT_KEYS if config.get(key)), 0)
return count, int(config.get("num_experts_per_tok") or 0)
def _parse_mixed_precision(quantized_layers: dict,
model_type: str = "",
num_experts: int = 0,
top_k: int = 0) -> "tuple[str, int, dict, dict]":
"""Parse MIXED_PRECISION quantized_layers dict.
Returns ``(dominant_quant_type, dominant_group_size, layer_overrides,
layer_group_sizes)``.
``layer_overrides`` maps **every** quantized module name to its quant-type
string. Modules not listed in ``quantized_layers`` are unquantized (FP16);
``make_linear`` falls back to FP16 when a module_name is absent from
``layer_overrides``.
Block-scaled FP8 (``FP8_BLOCK_SCALES``) has no form in the graph; the loader
dequantizes those weights, so a dense module runs FP16 and an expert bank
runs FP16 when ``Fp16MoePlugin`` can route it, else requantized W4A16.
"""
from collections import Counter
def _is_fp8_block(algo: str) -> bool:
return "FP8_BLOCK" in algo
def _mixed_quant_type(algo: str) -> str:
if _is_fp8_block(algo):
return QUANT_FP16
return _select_nvfp4_quant_type(_algo_to_quant_type(algo))
def _group_size(layer_config: dict, quant_type: str) -> int:
configured = int(layer_config.get("group_size", 1))
if configured != 1:
return configured
if quant_type == QUANT_MXFP8:
return 32
if quant_type in (QUANT_NVFP4, QUANT_NVFP4_A16):
return 16
return configured
algo_count: Counter = Counter()
algo_group_size: dict = {}
for layer_cfg in quantized_layers.values():
algo = layer_cfg.get("quant_algo", "").upper()
algo_count[algo] += 1
if algo not in algo_group_size:
algo_group_size[algo] = _group_size(layer_cfg,
_mixed_quant_type(algo))
if not algo_count:
return QUANT_FP16, 1, {}, {}
dominant_algo = algo_count.most_common(1)[0][0]
dominant_type = _mixed_quant_type(dominant_algo)
dominant_group_size = algo_group_size.get(dominant_algo, 1)
# Expand fused projection keys (``self_attn.qkv_proj``,
# ``mlp.gate_up_proj``) into the split names ``make_linear`` looks up
# (``q_proj``/``k_proj``/``v_proj`` and ``gate_proj``/``up_proj``).
layer_overrides: dict = {}
layer_group_sizes: dict = {}
for name, layer_cfg in quantized_layers.items():
algo = layer_cfg.get("quant_algo", "").upper()
short_name = _normalize_module_name(name)
quant_type = _mixed_quant_type(algo)
group_size = _group_size(layer_cfg, quant_type)
if (_is_muse_glimmer_model_type(model_type)
and short_name.endswith(".self_attn.output_gate_proj")):
short_name = short_name[:-len(".output_gate_proj")] + ".gate_proj"
if short_name.endswith(".self_attn.qkv_proj"):
prefix = short_name[:-len("qkv_proj")]
module_names = tuple(f"{prefix}{proj}"
for proj in ("q_proj", "k_proj", "v_proj"))
elif short_name.endswith(".mlp.gate_up_proj"):
prefix = short_name[:-len("gate_up_proj")]
module_names = tuple(f"{prefix}{proj}"
for proj in ("gate_proj", "up_proj"))
else:
module_names = (short_name, )
if _is_fp8_block(algo):
if (short_name.endswith(".mlp.experts")
and not _fp16_moe_routes(num_experts, top_k)):
quant_type, group_size = QUANT_NVFP4_A16, 16
else:
quant_type, group_size = QUANT_FP16, 1
for module_name in module_names:
layer_overrides[module_name] = quant_type
layer_group_sizes[module_name] = group_size
return (dominant_type, dominant_group_size, layer_overrides,
layer_group_sizes)
def _kv_norm(s: Optional[str]) -> Optional[str]:
if not s:
return None
return s.strip().lower() or None
def _safetensor_keys(model_dir: str) -> Optional[List[str]]:
"""Return all tensor keys from sharded or single-file safetensors."""
index_path = os.path.join(model_dir, "model.safetensors.index.json")
if os.path.exists(index_path):
with open(index_path) as f:
return list(json.load(f).get("weight_map", {}).keys())
st_path = os.path.join(model_dir, "model.safetensors")
if not os.path.exists(st_path):
return None
from safetensors import safe_open
with safe_open(st_path, framework="pt") as f:
return list(f.keys())
def _detect_visual_mha_fp8(model_dir: str) -> Optional[str]:
"""Return ``"fp8"`` iff the checkpoint carries visual-MHA Q/K/V scales.
Self-describing detection: the presence of ``<visual_prefix>*.q_scale``
buffers in the safetensors is the ground-truth signal that ViT FP8
MHA was calibrated. Avoids trusting modelopt's hf_quant_config.json
metadata (which has no ``visual_mha_quant_algo`` slot and conflates
ViT q/k/v_bmm with the LLM KV cache).
"""
keys = _safetensor_keys(model_dir)
if keys is None:
return None
for k in keys:
if k.endswith(".q_scale") and any(h in k for h in VISUAL_PREFIXES):
return "fp8"
return None
def _detect_llm_kv_cache_fp8(model_dir: str) -> Optional[str]:
"""Return ``"fp8"`` iff the checkpoint carries LLM KV-cache K/V scales.
Self-describing detection: an ``<llm_attn>.k_proj.k_scale`` buffer
outside of any visual / vision subtree is the ground-truth signal.
Modelopt's ``hf_quant_config.json:kv_cache_quant_algo`` field would
say "FP8" even when only ViT MHA was requested (because its writer
trips on any enabled ``k_bmm_quantizer``, including the visual one);
ignoring that field and looking at the checkpoint directly avoids
the false positive.
"""
keys = _safetensor_keys(model_dir)
if keys is None:
return None
for k in keys:
if k.endswith(".k_proj.k_scale") and not any(h in k
for h in VISUAL_PREFIXES):
return "fp8"
return None