# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Checkpoint metadata I/O and runtime sidecars next to exported ONNX.
Weights are loaded only via :func:`loader.load_weights`.
"""
from __future__ import annotations
import json
import logging
import math
import os
import re
import shutil
from typing import TYPE_CHECKING, Any, Dict, Tuple
if TYPE_CHECKING:
from ..models.default.modeling_default import CausalLM
logger = logging.getLogger(__name__)
__all__ = [
"RUNTIME_TOKENIZER_FILENAMES",
"validate_external_speculative_target",
"normalize_rope_scaling_for_runtime",
"rotary_dim_for_runtime",
"load_checkpoint_config_dicts",
"load_config_dict",
"build_runtime_llm_config_dict",
"write_runtime_artifacts",
]
RUNTIME_TOKENIZER_FILENAMES: Tuple[str, ...] = (
"tokenizer.json",
"tokenizer_config.json",
"tokenizer.model",
"special_tokens_map.json",
)
def validate_external_speculative_target(config: Dict[str, Any],
*,
mode: str = "") -> None:
"""Reject unsupported external speculative verifier architectures."""
model_type = str(config.get("model_type", "")).lower()
if model_type == "nemotron_h" and mode in ("dflash", "dspark",
"dspark_onnx"):
return
if model_type == "nemotron_h_puzzle" and mode == "lilicorr":
return
if model_type.startswith("nemotron"):
raise ValueError(
"Nemotron external speculative decoding is not supported")
def normalize_rope_scaling_for_runtime(rope_scaling: Any) -> Any:
"""Normalize HF MRoPE / rope_parameters metadata to the shape expected by
the C++ runtime.
"""
if not isinstance(rope_scaling, dict):
return rope_scaling
normalized = dict(rope_scaling)
if "mrope_section" in normalized:
rope_type = normalized.get("type") or normalized.get("rope_type")
if rope_type in (None, "default", "mrope"):
normalized["type"] = "default"
normalized["rope_type"] = "default"
# rope_parameters (transformers v5) carries "rope_type" without "type";
# propagate the alias so Python callers keyed off "type" still work.
# C++ collectRopeConfig() accepts either key.
if "type" not in normalized and "rope_type" in normalized:
normalized["type"] = normalized["rope_type"]
return normalized
def rotary_dim_for_runtime(rope_config: Dict[str, Any], head_dim: int,
fallback_partial_rotary_factor: float) -> int:
"""Return the RoPE binding width expected by the C++ builder/runtime."""
rope_scaling = rope_config.get("rope_scaling")
if isinstance(rope_scaling, dict):
rope_type = str(
rope_scaling.get("type") or rope_scaling.get("rope_type") or "")
if rope_type in ("default", "proportional"):
return int(head_dim)
partial_rotary_factor = float(
rope_config.get("partial_rotary_factor",
fallback_partial_rotary_factor))
return int(float(head_dim) * partial_rotary_factor)
def _normalize_explicit_rope_config_for_runtime(
rope_config: Dict[str, Any]) -> Dict[str, Any]:
"""Normalize one explicit runtime RoPE config block."""
normalized = dict(rope_config)
normalized["rope_scaling"] = normalize_rope_scaling_for_runtime(
normalized.get("rope_scaling"))
return normalized
def _torch_dtype_to_config_str(dtype: Any) -> str:
"""Map a ``torch.dtype`` to the string token the runtime parser accepts
(see cpp/runtime/config/llmEngineConfig.cpp::parseStateDtype).
Deferred import of torch so importing this module does not drag it in on
pure-CPU tooling paths.
"""
import torch
mapping = {
torch.float16: "fp16",
torch.float32: "fp32",
torch.bfloat16: "bf16",
}
if dtype not in mapping:
raise ValueError(
f"No config-string mapping for torch dtype {dtype!r}. "
f"Supported: {sorted(v for v in mapping.values())}")
return mapping[dtype]
def _nested_config_to_dict(sub: Any) -> Dict[str, Any]:
if isinstance(sub, dict):
return sub
if hasattr(sub, "to_dict"):
return sub.to_dict()
return {}
def _promote_llm_subconfig(config: Any, root: Dict[str,
Any]) -> Dict[str, Any]:
"""Return the dict used for LLM architecture fields (text / nested block)."""
if root.get("num_attention_heads") is not None:
return root
# Qwen3-ASR / Qwen3-Omni: LLM lives at thinker_config.text_config.
# Resolve the thinker/talker wrappers BEFORE the generic top-level loop:
# newer transformers can expose a spurious top-level ``text_config`` (with
# the wrong hidden_size) alongside the real thinker_config.text_config, and
# the generic loop would otherwise pick that wrong one.
thinker = root.get("thinker_config")
if isinstance(thinker, dict):
for name in ("text_config", "llm_config", "language_config"):
sub_dict = _nested_config_to_dict(thinker.get(name, {}))
if (sub_dict.get("hidden_size") is not None
and sub_dict.get("num_attention_heads") is not None):
return sub_dict
# Qwen3-TTS: LLM (talker) lives at talker_config
talker = root.get("talker_config")
if isinstance(talker, dict):
if (talker.get("hidden_size") is not None
and talker.get("num_attention_heads") is not None):
return talker
# Standard VLMs (e.g. Qwen2.5-VL): LLM is a top-level sub-config.
for name in ("llm_config", "text_config", "language_config"):
sub = getattr(config, name, None)
if sub is None and name in root:
sub = root[name]
sub_dict = _nested_config_to_dict(sub)
if (sub_dict.get("hidden_size") is not None
and sub_dict.get("num_attention_heads") is not None):
return sub_dict
return root
def _promote_alpamayo_llm_config(root: Dict[str, Any]) -> Dict[str, Any]:
"""For Alpamayo-R1: load the full VLM text config from ``vlm_name_or_path``.
The Alpamayo root config.json is flat and does not embed VLM architecture
fields. We load ``AutoConfig.from_pretrained(vlm_name_or_path)`` to get
the full Qwen3-VL config and then promote its text sub-config (which
contains ``num_attention_heads``, ``hidden_size``, etc.).
"""
from transformers import AutoConfig
# vlm_name_or_path is a default in AlpamayoR1Config, not persisted in
# config.json. Fall back to the known default for Alpamayo-R1.
vlm_name = root.get("vlm_name_or_path", "Qwen/Qwen3-VL-8B-Instruct")
if not vlm_name:
logger.warning("alpamayo_r1 config missing vlm_name_or_path; "
"falling back to root config")
return root
try:
vlm_cfg = AutoConfig.from_pretrained(vlm_name, trust_remote_code=True)
vlm_dict = vlm_cfg.to_dict()
except (ValueError, OSError) as exc:
logger.warning(
"Failed to load VLM config from %s (%s); "
"falling back to root config", vlm_name, exc)
return root
# The VLM config (e.g. Qwen3-VL) has a text sub-config at text_config
# or language_config. Promote it using the existing helper.
llm = _promote_llm_subconfig(vlm_cfg, vlm_dict)
# Alpamayo extends the vocabulary with trajectory tokens; the root
# config carries the true vocab_size which must override the base VLM's.
if root.get("vocab_size") is not None:
llm["vocab_size"] = root["vocab_size"]
# Qwen3-VL text_config stores mRoPE info under ``rope_parameters``
# rather than ``rope_scaling``. Promote it so downstream code and the
# C++ runtime find it under the expected ``rope_scaling`` key.
if not llm.get("rope_scaling") and llm.get("rope_parameters"):
llm["rope_scaling"] = llm["rope_parameters"]
return llm
# TODO: remove together with the pruning in load_checkpoint_config_dicts once
# dense Qwen4 ships its own HF config class (no MoE defaults left to strip).
_MOE_COUNT_KEYS = ("num_experts", "num_local_experts", "n_routed_experts",
"moe_num_experts")
_MOE_SIZING_KEYS = _MOE_COUNT_KEYS + (
"num_experts_per_tok", "moe_intermediate_size",
"shared_expert_intermediate_size", "decoder_sparse_step")
[docs]
def load_checkpoint_config_dicts(
model_dir: str) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Return ``(root_dict, llm_dict)`` from the checkpoint config.
Tries ``AutoConfig.from_pretrained`` first (handles registered HF model
types). Falls back to reading ``config.json`` directly for custom /
not-yet-registered model types (e.g. ``qwen3_asr``, ``qwen3_tts``).
For multimodal models (e.g. Qwen2.5-VL, Qwen3-ASR), the LLM text config
is promoted out of the nested sub-object by :func:`_promote_llm_subconfig`.
Any fields lost during promotion are patched back from the raw JSON.
"""
from transformers import AutoConfig
raw_path = os.path.join(model_dir, "config.json")
raw: Dict[str, Any] = {}
if os.path.exists(raw_path):
with open(raw_path) as _f:
raw = json.load(_f)
elif not os.path.isdir(model_dir):
# model_dir is likely an HF model ID (e.g. "Qwen/Qwen3-ASR-0.6B").
# Download config.json from HF Hub so the raw fallback works.
try:
from huggingface_hub import hf_hub_download
local = hf_hub_download(model_dir, "config.json")
with open(local) as _f:
raw = json.load(_f)
except (OSError, ImportError, ValueError):
pass
try:
config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
root = config.to_dict()
except (ValueError, OSError, AttributeError) as exc:
# Unknown / not-yet-registered model type — fall back to raw JSON.
# transformers' RoPE validation raises AttributeError for DeepSpec
# Gemma4 drafts that mix scalar and per-layer-type rope_parameters.
logger.warning(
"AutoConfig.from_pretrained failed for %s (%s); "
"falling back to raw config.json.",
model_dir,
exc,
)
root = raw
config = root
speculator_type = raw.get("speculators_model_type")
transformer_layer_config = raw.get("transformer_layer_config")
if (speculator_type in ("eagle3", "dflash", "dspark", "peagle")
and isinstance(transformer_layer_config, dict)):
# Speculators checkpoints keep the decoder architecture in a dedicated
# sub-config and the algorithm contract at the root.
root = raw
llm = dict(transformer_layer_config)
for key, value in raw.items():
if key not in llm and value is not None:
llm[key] = value
if speculator_type in ("eagle3", "peagle"):
target_layers = raw.get("eagle_aux_hidden_state_layer_ids")
if target_layers:
llm["target_layer_ids"] = list(target_layers)
elif speculator_type == "dflash":
aux_layers = [
int(layer)
for layer in raw.get("aux_hidden_state_layer_ids") or []
]
if any(layer <= 0 for layer in aux_layers):
raise ValueError(
"DFlash auxiliary hidden-state layer IDs must be positive")
dflash_config = {
"target_layer_ids": [layer - 1 for layer in aux_layers],
"block_size": raw.get("block_size", 8),
}
if raw.get("mask_token_id") is not None:
dflash_config["mask_token_id"] = raw["mask_token_id"]
llm["dflash_config"] = dflash_config
else:
aux_layers = [
int(layer)
for layer in raw.get("aux_hidden_state_layer_ids") or []
]
if any(layer <= 0 for layer in aux_layers):
raise ValueError(
"DSpark auxiliary hidden-state layer IDs must be positive")
dspark_config = {
"target_layer_ids": [layer - 1 for layer in aux_layers],
"block_size":
raw.get("block_size", 8),
"enable_confidence_head":
raw.get("enable_confidence_head", False),
"confidence_head_with_markov":
raw.get("confidence_head_with_markov", False),
"markov_head_type":
raw.get("markov_head_type", ""),
"markov_rank":
raw.get("markov_rank", 0),
}
if raw.get("mask_token_id") is not None:
dspark_config["mask_token_id"] = raw["mask_token_id"]
llm["dspark_config"] = dspark_config
# Alpamayo-R1: flat config with no embedded VLM sub-config.
# Load the full VLM architecture config from vlm_name_or_path and promote
# the text sub-config so downstream sees a standard Qwen3-VL text config.
elif root.get("model_type") == "alpamayo_r1":
llm = _promote_alpamayo_llm_config(root)
else:
llm = _promote_llm_subconfig(config, root)
# Patch: for multimodal models where AutoConfig loses top-level fields,
# merge them in from the raw config.json. Only fields absent from llm
# are copied; existing llm fields are never overwritten.
for key, val in raw.items():
if key not in llm and val is not None:
llm[key] = val
# Sliding-window compatibility: HF attention configs (e.g. Qwen3Config) null
# out sliding_window unless the legacy use_sliding_window flag is set, but
# checkpoints on the newer convention declare per-layer "sliding_attention"
# in layer_types and omit that flag. The patch-back above only fills ABSENT
# keys, so the nulled window would survive. Treat layer_types as the
# authority and recover the raw value; a checkpoint that deliberately
# disables its window reports full_attention layers and is untouched.
if llm.get("sliding_window") is None:
for source in (llm, raw):
layer_types = source.get("layer_types")
if not isinstance(layer_types, list):
continue
if "sliding_attention" not in layer_types:
continue
raw_window = raw.get("sliding_window")
if raw_window is None:
sub = raw.get("text_config")
raw_window = sub.get("sliding_window") if isinstance(
sub, dict) else None
if raw_window is not None:
llm["sliding_window"] = raw_window
break
architectures = llm.get("architectures") or raw.get("architectures") or []
if (any("dspark" in str(name).lower() for name in architectures)
and not llm.get("dspark_config")
and isinstance(llm.get("dflash_config"), dict)):
llm["dspark_config"] = dict(llm["dflash_config"])
# Recover fields from nested text sub-config that AutoConfig.to_dict()
# drops (e.g. Gemma4TextConfig drops global_head_dim).
for subkey in ("text_config", "llm_config", "language_config"):
raw_sub = raw.get(subkey)
if isinstance(raw_sub, dict):
for key, val in raw_sub.items():
if key not in llm and val is not None:
llm[key] = val
break
# AutoConfig fills class defaults; keep expert sizing only when the
# checkpoint itself declares an expert count.
# TODO: remove once dense Qwen4 ships its own HF config class.
if raw:
declared = (raw, _promote_llm_subconfig(raw, raw),
raw.get("transformer_layer_config") or {})
if not any(
source.get(key) for source in declared
for key in _MOE_COUNT_KEYS):
for key in _MOE_SIZING_KEYS:
llm.pop(key, None)
# VLM / transformers v5 rope compatibility: rope_scaling may be null
# while rope_parameters carries the real config (transformers v5
# convention), and either may live only in a nested sub-config
# (e.g. text_config for Qwen3-VL). Recover into rope_scaling so
# collectRopeConfig() in C++ detects kMRope correctly and Python
# readers that key off rope_scaling (e.g. longrope in
# build_runtime_llm_config_dict) still work.
if not llm.get("rope_scaling"):
candidate = llm.get("rope_parameters")
if not candidate:
for subkey in ("text_config", "language_config", "llm_config"):
raw_sub = raw.get(subkey) or {}
if isinstance(raw_sub, dict):
candidate = (raw_sub.get("rope_scaling")
or raw_sub.get("rope_parameters"))
if candidate:
break
if isinstance(candidate, dict):
llm["rope_scaling"] = candidate
if llm.get("rope_scaling"):
llm["rope_scaling"] = normalize_rope_scaling_for_runtime(
llm["rope_scaling"])
return root, llm
def _alpamayo_kv_cache_scales(model: "CausalLM", root_cfg: Dict[str, Any],
cfg_json: Dict[str, Any]):
"""Return Alpamayo's per-layer FP8 K/V dequant scales, if required.
The ONNX export path caches each attention layer's calibrated ``[q, k,
v]`` values on ``_qkv_scales_float`` before tracing. The action engine is
an external FP16 KV consumer, so its runtime sidecar must retain the K/V
entries instead of leaving them only inside the serialized LLM plugins.
"""
if (root_cfg.get("model_type") != "alpamayo_r1"
or cfg_json.get("kv_cache_dtype") != "fp8"):
return None
layer_scales = {}
for name, module in model.named_modules():
if not getattr(module, "enable_fp8_kv_cache", False):
continue
if any(component in name
for component in ("visual", "vision", "action", "expert")):
continue
match = re.search(r"(?:^|\.)layers\.(\d+)\.self_attn$", name)
if match is None:
continue
layer = int(match.group(1))
if layer in layer_scales:
raise ValueError(
f"Alpamayo FP8 KV export found duplicate scales for decoder layer {layer}"
)
qkv_scales = getattr(module, "_qkv_scales_float", None)
if qkv_scales is None or len(qkv_scales) != 3:
raise ValueError(
"Alpamayo FP8 KV export is missing a Q/K/V scale triple")
pair = [float(qkv_scales[1]), float(qkv_scales[2])]
if not all(math.isfinite(scale) and scale > 0.0 for scale in pair):
raise ValueError(
"Alpamayo FP8 KV dequant scales must be finite and positive")
layer_scales[layer] = pair
expected_layers = int(cfg_json["num_hidden_layers"])
expected_indices = set(range(expected_layers))
if set(layer_scales) != expected_indices:
raise ValueError(
"Alpamayo FP8 KV export found "
f"scales for decoder layers {sorted(layer_scales)}, expected "
f"{sorted(expected_indices)}")
return [layer_scales[layer] for layer in range(expected_layers)]
[docs]
def load_config_dict(model_dir: str) -> Dict[str, Any]:
"""Return only the promoted LLM config dict."""
return load_checkpoint_config_dicts(model_dir)[1]
def _export_tool_version() -> str:
"""Version string for runtime ``config.json`` (``edgellm_version`` field)."""
from .._version import __version__
return __version__
def _determine_spec_decode_type(config) -> str:
"""Return the speculative decoding algorithm for runtime config."""
if config.gemma4_mtp_base or config.gemma4_mtp_draft:
return "gemma4_mtp"
if config.is_eagle3_draft or config.eagle_base:
return "eagle3"
if config.is_domino_draft or config.domino_base:
return "domino"
if config.is_dflash_draft or config.dflash_base:
return "dflash"
if config.is_jetspec_draft or config.jetspec_base:
return "jetspec"
if config.is_dspark_draft or config.dspark_base:
return "dspark"
if config.is_mtp_draft or config.mtp_base:
return "mtp"
return "none"
def _determine_engine_role(config) -> str:
"""Return the engine role within the speculative decoding deployment."""
if (config.is_eagle3_draft or config.is_dflash_draft
or config.is_jetspec_draft or config.is_dspark_draft
or config.is_mtp_draft or config.gemma4_mtp_draft):
return "draft"
if (config.eagle_base or config.dflash_base or config.jetspec_base
or config.dspark_base or config.mtp_base
or config.gemma4_mtp_base):
return "base"
return "llm"
_GLOBAL_MODEL_FIELDS = (
"vocab_size",
"hidden_size",
"intermediate_size",
"num_hidden_layers",
"num_attention_heads",
"num_key_value_heads",
"head_dim",
"max_position_embeddings",
"rope_theta",
"partial_rotary_factor",
"num_deepstack_features",
)
_RANK_LOCAL_OVERRIDE_FIELDS = (
"num_attention_heads",
"num_key_value_heads",
"intermediate_size",
)
def _merge_rank_config(existing_rank_configs: Any,
rank_config: Dict[str, Any]) -> list:
merged = []
if isinstance(existing_rank_configs, list):
for existing in existing_rank_configs:
if not isinstance(existing, dict):
continue
if existing.get("rank") == rank_config["rank"]:
continue
sanitized = dict(existing)
sanitized.pop("engine", None)
merged.append(sanitized)
merged.append(rank_config)
return sorted(merged, key=lambda item: int(item.get("rank", 0)))
def _apply_global_model_fields(out: Dict[str, Any],
global_llm_config: Any) -> None:
if not isinstance(global_llm_config, dict):
return
for field in _GLOBAL_MODEL_FIELDS:
if field in out and global_llm_config.get(field) is not None:
out[field] = global_llm_config[field]
def _tp_global_kv_layer_configs(local_configs: list, tp_size: int) -> list:
global_configs = []
for layer in local_configs:
if layer is None:
global_configs.append(None)
continue
global_layer = dict(layer)
if "num_kv_heads" in global_layer:
global_layer["num_kv_heads"] *= tp_size
global_configs.append(global_layer)
return global_configs
def build_runtime_llm_config_dict(
model: "CausalLM",
global_llm_config: Any = None,
existing_rank_configs: Any = None) -> Dict[str, Any]:
"""JSON object written as the runtime config beside the ONNX export.
Top-level model fields describe the original/global model. Multi-rank
exports attach rank-local ONNX shard metadata and sharded shape overrides
under ``rank_configs`` so the shared ``config_world{N}.json`` stays
self-describing without engine-build metadata.
"""
config = model.config
mc = config.mamba_cfg
rope_scaling = normalize_rope_scaling_for_runtime(config.rope_scaling)
mapping = getattr(config, "mapping", None)
tp_size = max(1, getattr(config, "tp_size", getattr(mapping, "tp_size",
1)))
tp_rank = max(0, getattr(config, "tp_rank", getattr(mapping, "tp_rank",
0)))
world_size = max(1, getattr(mapping, "world_size", tp_size))
rank = max(0, getattr(mapping, "rank", tp_rank))
diffusion_engine_role = getattr(model, "diffusion_engine_role", None)
if config.is_diffusion_gemma and diffusion_engine_role is None:
diffusion_engine_role = "dllm"
out: Dict[str, Any] = {
"model": ("diffusion_gemma_text"
if config.is_diffusion_gemma else config.model_type),
"spec_decode_type":
_determine_spec_decode_type(config),
"engine_role":
diffusion_engine_role or _determine_engine_role(config),
"edgellm_version":
_export_tool_version(),
"vocab_size":
config.vocab_size,
"hidden_size":
config.hidden_size,
"intermediate_size":
config.intermediate_size,
"num_hidden_layers":
config.num_hidden_layers,
"num_attention_heads":
config.num_attention_heads,
"num_key_value_heads":
config.num_key_value_heads,
"head_dim":
config.head_dim,
"max_position_embeddings":
config.max_position_embeddings,
"rope_theta":
config.rope_theta,
"rope_scaling":
rope_scaling,
"partial_rotary_factor":
config.partial_rotary_factor,
"num_deepstack_features":
config.num_deepstack_features,
"use_vision_bidirectional_attention":
bool(config.use_vision_bidirectional_attention
and not (config.eagle_base or config.dflash_base
or config.jetspec_base or config.dspark_base
or config.mtp_base or config.gemma4_mtp_base
or config.is_eagle3_draft or config.is_dflash_draft
or config.is_jetspec_draft or config.is_dspark_draft
or config.is_mtp_draft or config.gemma4_mtp_draft)),
"rms_norm_eps":
float(config.rms_norm_eps),
}
if config.attention_scaling:
out["attention_scaling"] = float(config.attention_scaling)
if config.final_logit_softcapping is not None:
out["final_logit_softcapping"] = float(config.final_logit_softcapping)
if config.attention_layer_types:
out["attention_layer_types"] = list(config.attention_layer_types)
if config.sliding_window_size >= 0:
out["sliding_window_size"] = int(config.sliding_window_size)
if config.attention_k_eq_v:
out["attention_k_eq_v"] = True
if config.num_global_key_value_heads:
out["num_global_key_value_heads"] = int(
config.num_global_key_value_heads)
if config.num_experts > 0:
out.update({
"num_experts": int(config.num_experts),
"num_experts_per_tok": int(config.num_experts_per_tok),
"moe_intermediate_size": int(config.moe_intermediate_size),
"enable_moe_block": bool(config.enable_moe_block),
})
if config.is_diffusion_gemma:
diffusion_cfg = (config.diffusion.to_dict()
if config.diffusion is not None else {})
diffusion_unified_conditioning = bool(
getattr(model, "diffusion_unified_conditioning", False))
if diffusion_engine_role == "dllm" and not diffusion_unified_conditioning:
raise ValueError(
"DiffusionGemma dllm export requires unified self-conditioning. "
"Use DiffusionGemmaBackbone so the self-conditioning module is "
"created with the backbone before writing runtime config.")
out.update({
"decoding_strategy":
"block_diffusion",
"diffusion_config":
diffusion_cfg,
"self_conditioning_size":
config.self_conditioning_size,
"diffusion_unified_conditioning":
diffusion_unified_conditioning,
"context_mask_selector_enabled": (diffusion_engine_role == "dllm"),
"diffusion_engines": {
"dllm": {
"path": "dllm.engine",
"role": "dllm",
},
},
})
attention_layer_types = (getattr(config, "attention_layer_types", []) if
(str(config.model_type).startswith("gemma4")
or config.is_diffusion_gemma) else [])
# Heterogeneous head dimensions (e.g. Gemma4: sliding=256, global=512)
if config.global_head_dim and config.global_head_dim != config.head_dim:
out["global_head_dim"] = config.global_head_dim
use_global_kv_heads = bool(config.attention_k_eq_v
and config.num_global_key_value_heads)
# C++ sizes KV tensors from kv_layer_configs below. Keep the top-level
# field only for Python/config round-trip metadata.
if (config.num_global_key_value_heads and use_global_kv_heads
and config.num_global_key_value_heads
!= config.num_key_value_heads):
out["num_global_key_value_heads"] = config.num_global_key_value_heads
# Per-layer-type RoPE: extract global attention RoPE parameters from
# rope_scaling.full_attention (Gemma4: theta=1000000, prf=0.25).
full_attn_rope = (rope_scaling or {}).get("full_attention", {})
if full_attn_rope.get("rope_theta"):
out["global_rope_theta"] = float(full_attn_rope["rope_theta"])
# KV-sharing donors: shared layers read from donor layer's KV cache.
num_kv_shared = getattr(config, "num_kv_shared_layers", 0)
if num_kv_shared > 0 and not config.gemma4_mtp_draft:
from ..models.gemma4.modeling_gemma4_text import \
_compute_kv_donor_indices
donor_map = _compute_kv_donor_indices(config)
# Build donors array of length num_hidden_layers (all are attention).
# -1 = no sharing, otherwise = donor layer index.
donors = [-1] * config.num_hidden_layers
for shared_idx, donor_idx in donor_map.items():
donors[shared_idx] = donor_idx
out["kv_sharing_donors"] = donors
ple_enabled = config.hidden_size_per_layer_input > 0
out["gemma_ple_enabled"] = ple_enabled
out["gemma_ple_layer_count"] = config.num_hidden_layers if ple_enabled else 0
out["gemma_ple_hidden_size"] = (config.hidden_size_per_layer_input
if ple_enabled else 0)
# Qwen4-Exp n-gram injection. Unrelated to the Gemma4 fields above despite the shared name;
# these describe a hashed n-gram table the runtime gathers from, not per-layer embeddings.
# Absent entirely when no layer asks for it, so that the runtime builds no state and binds
# no tensor for a model without one.
if config.ngram_layers:
out["ngram_layers"] = config.ngram_layers
out["ngram_embed_dim"] = config.ple_embed_dim
out["ngram_conv_kernel"] = config.ple_conv_kernel_size
# The injection writes into every residual stream at once, so its conv spans the full
# multi-stream width. Emitted directly: the runtime needs the width, not the reason.
out["ngram_conv_dim"] = config.hc_count * config.hidden_size
out["ngram_size"] = config.ngram_size
out["ngram_heads_per_order"] = config.heads_per_ngram
# Read from the model class, as the hybrid state dtypes below are: that attribute is what
# `export_onnx` builds the dummy state tensor from, which fixes the engine binding dtype.
# Separate from `conv_state_dtype`, which is only emitted for models with recurrent layers.
ngram_conv_dtype = getattr(model, "NGRAM_CONV_STATE_DTYPE", None)
if ngram_conv_dtype is None:
raise AttributeError(
f"{type(model).__name__} declares ngram_layers but does not expose "
f"NGRAM_CONV_STATE_DTYPE. Add a class-level NGRAM_CONV_STATE_DTYPE "
f"(torch.dtype) matching the dtype of the n-gram conv state tensor its "
f"export_onnx builds.")
out["ngram_conv_state_dtype"] = _torch_dtype_to_config_str(
ngram_conv_dtype)
# longrope requires original_max_position_embeddings for scaling factor computation.
if (isinstance(rope_scaling, dict)
and rope_scaling.get("type") == "longrope"
and config.original_max_position_embeddings is not None):
out["original_max_position_embeddings"] = config.original_max_position_embeddings
if config.use_dual_rope:
out["sliding_rope_config"] = _normalize_explicit_rope_config_for_runtime(
config.sliding_rope_config or {})
out["full_rope_config"] = _normalize_explicit_rope_config_for_runtime(
config.full_rope_config or {})
if config.gemma4_mtp_draft:
out["sliding_rotary_dim"] = rotary_dim_for_runtime(
out["sliding_rope_config"], config.head_dim, 1.0)
out["full_rotary_dim"] = rotary_dim_for_runtime(
out["full_rope_config"],
config.global_head_dim or config.head_dim,
config.partial_rotary_factor,
)
if config.is_hybrid and mc is not None:
out.update({
"num_linear_attn_layers":
config.num_mamba_layers,
"num_attention_layers":
config.num_attn_layers,
"recurrent_state_num_heads":
mc.num_heads,
"recurrent_state_head_dim":
mc.head_dim,
"recurrent_state_size":
mc.ssm_state_size,
"recurrent_state_num_groups":
mc.n_groups,
"recurrent_spec_verify_mode":
"replay",
"conv_dim":
mc.conv_dim,
"conv_kernel":
mc.conv_kernel,
# Nemotron-H attention is NoPE
"use_rope":
config.num_attn_layers > 0 and not config.is_nemotron_h,
})
gc = config.gdn_cfg
if config.is_hybrid and gc is not None:
out.update({
"num_linear_attn_layers": config.num_gdn_layers,
"num_attention_layers": config.num_attn_layers,
"recurrent_state_num_heads": gc.num_value_heads,
"recurrent_state_head_dim": gc.key_head_dim,
"recurrent_state_size": gc.value_head_dim,
"recurrent_spec_verify_mode": "snapshot",
"conv_dim": gc.conv_dim,
"conv_kernel": gc.conv_kernel,
"use_rope": config.num_attn_layers > 0,
})
if (not config.is_hybrid
and config.num_attn_layers != config.num_hidden_layers):
out["num_attention_layers"] = config.num_attn_layers
# Emit canonical per-layer config consumed by the C++ HybridCacheManager.
# Only attention and linear-attention layers carry KV/recurrent state and
# must appear in the per-layer routing table. MLP/MoE layers are skipped.
# Pure-attention Gemma4 models also need this table when sliding layers use
# a different cache capacity.
_emit_kv_table = config.layer_types and (
config.is_hybrid or config.num_attn_layers != config.num_hidden_layers
or attention_layer_types or config.indexer_head_dim > 0)
if _emit_kv_table:
from ..config import (_VALID_ATTENTION_LAYER_TYPES, LAYER_ATTN,
LAYER_GDN, LAYER_MAMBA)
gemma4_kv_layer_config = None
if attention_layer_types:
from ..models.gemma4.modeling_gemma4_text import \
_gemma4_kv_layer_config as gemma4_kv_layer_config
# ``config.layer_types`` normalizes recurrent layers to LAYER_GDN /
# LAYER_MAMBA, but attention layers keep their raw HF type (e.g.
# Qwen3.5 uses ``"full_attention"``), so match the attention family
# explicitly. Matching only LAYER_ATTN silently drops those layers,
# collapsing the per-layer routing table and shifting every attention
# layer's position (see num_attn_layers, which counts the same set).
attention_types = (LAYER_ATTN, ) + _VALID_ATTENTION_LAYER_TYPES
normalized_layer_types: list = []
kv_layer_configs: list = []
for layer_idx, lt in enumerate(config.layer_types):
if lt in attention_types:
normalized_layer_types.append("attention")
if gemma4_kv_layer_config is not None:
layer_config = gemma4_kv_layer_config(config, layer_idx)
else:
layer_config = {
"num_kv_heads": config.num_key_value_heads,
"head_dim": config.kv_pool_head_dim,
}
kv_layer_configs.append(layer_config)
elif lt in (LAYER_MAMBA, LAYER_GDN):
normalized_layer_types.append("mamba")
kv_layer_configs.append(None)
# Non-stateful layers (e.g. LAYER_MLP) have no cache slot and are
# intentionally omitted from the per-layer routing table.
out["layer_types"] = normalized_layer_types
out["kv_layer_configs"] = kv_layer_configs
# Declares that every attention layer runs Qwen Sparse Attention (see
# kv_pool_head_dim), which the runtime's compact capability check refuses.
if config.indexer_head_dim > 0:
out["indexer_head_dim"] = config.indexer_head_dim
if config.is_eagle3_draft:
draft_vocab = config.draft_vocab_size or config.vocab_size
target_hidden = config.eagle3_target_hidden_size
eagle3_config = {
"target_layer_ids": list(config.eagle3_target_layer_ids),
"num_target_layers": config.eagle3_num_target_layers,
}
if config.eagle_fc_norm:
eagle3_config["fc_norm"] = True
if config.eagle_norm_output:
eagle3_config["norm_output"] = True
if config.draft_candidate_count > 0:
eagle3_config["candidate_head"] = {
"type": "low_rank",
"rank": config.draft_candidate_rank,
"num_candidates": config.draft_candidate_count,
}
out.update({
"draft_vocab_size": draft_vocab,
"base_model_hidden_size":
target_hidden * config.eagle3_num_target_layers,
"eagle3_config": eagle3_config,
})
if config.is_mtp_draft:
# draft_vocab_size stays the full vocab; reduced_vocab_size, if set,
# narrows only the logits width. The base feeds back its residual,
# which is hc_count streams wide on a multi-stream model.
out.update({
"draft_vocab_size":
config.vocab_size,
"base_model_hidden_size":
config.hidden_size * max(config.hc_count, 1),
})
if config.gemma4_mtp_base:
out.update({
"base_model_hidden_size":
config.hidden_size,
"layer_types":
list(config.raw_layer_types or config.layer_types),
"sliding_window":
config.sliding_window_size,
"global_head_dim":
config.global_head_dim,
"num_kv_shared_layers":
config.num_kv_shared_layers,
"rope_parameters":
normalize_rope_scaling_for_runtime(config.rope_parameters),
"attention_k_eq_v":
config.attention_k_eq_v,
})
if config.gemma4_mtp_draft:
out.update({
"model":
"gemma4_assistant",
"draft_vocab_size":
config.vocab_size,
"base_model_hidden_size":
config.backbone_hidden_size,
"assistant_hidden_size":
config.assistant_hidden_size or config.hidden_size,
"shares_target_kv":
config.shares_target_kv,
"has_own_kv_cache":
config.has_own_kv_cache,
"constant_draft_positions":
config.constant_draft_positions,
"returns_feedback_hidden":
config.returns_feedback_hidden,
"use_ordered_embeddings":
config.use_ordered_embeddings,
"num_centroids":
config.num_centroids,
"centroid_intermediate_top_k":
config.centroid_intermediate_top_k,
"sparse_logits_enabled":
config.sparse_logits_enabled,
"layer_types":
list(config.raw_layer_types or config.layer_types),
"sliding_window":
config.sliding_window_size,
"global_head_dim":
config.global_head_dim,
"num_kv_shared_layers":
config.num_kv_shared_layers,
"rope_parameters":
normalize_rope_scaling_for_runtime(config.rope_parameters),
"attention_k_eq_v":
config.attention_k_eq_v,
"kv_sharing_map":
list(config.kv_sharing_map),
})
is_dflash = config.is_dflash_draft or config.dflash_base
if is_dflash:
version = int(config.dflash_version)
dflash = {
"version": version,
"target_layer_ids": list(config.dflash_target_layer_ids),
"block_size": config.dflash_block_size,
"mask_token_id": config.dflash_mask_token_id,
"supports_probabilistic_sampling": version == 2,
}
is_lilicorr = config.dflash_projector_type == "lilicorr"
if version == 2:
dflash.update({
"is_causal": bool(config.dflash2_is_causal),
"conv_kernel_size": config.dflash2_conv_kernel_size,
"conv_group_size": config.dflash2_conv_group_size,
})
if not is_lilicorr:
dflash.update({
"selector_rank": config.dflash2_selector_rank,
"selector_top_k": config.dflash2_selector_top_k,
"selector_file": "dflash2_selector.safetensors",
})
if is_lilicorr:
dflash.update({
"projector_type":
"lilicorr",
"lilicorr_candidate_topk":
config.lilicorr_candidate_topk,
})
if config.dflash_projector_type == "domino":
dflash.update({
"projector_type": "domino",
"gru_hidden_dim": config.domino_gru_hidden_dim,
"proj_dim": config.domino_proj_dim,
"prefix_len": config.domino_prefix_len,
"pool_size": config.domino_pool_size,
"shift_label": config.domino_shift_label,
"heads_file": "domino_heads.safetensors",
})
out["dflash_config"] = dflash
if config.is_dflash_draft:
out.update({
"draft_vocab_size":
config.vocab_size,
"base_model_hidden_size":
len(config.dflash_target_layer_ids) * config.hidden_size,
})
if config.is_jetspec_draft:
out.update({
"draft_vocab_size":
config.vocab_size,
"base_model_hidden_size":
len(config.jetspec_target_layer_ids) * config.hidden_size,
"jetspec_config": {
"target_layer_ids": list(config.jetspec_target_layer_ids),
"block_size": config.jetspec_block_size,
"mask_token_id": config.jetspec_mask_token_id,
"causal_head": bool(config.jetspec_causal_head),
},
})
if config.jetspec_base:
out.update({
"jetspec_config": {
"target_layer_ids": list(config.jetspec_target_layer_ids),
"block_size": config.jetspec_block_size,
"mask_token_id": config.jetspec_mask_token_id,
"causal_head": bool(config.jetspec_causal_head),
},
})
if config.is_dspark_draft:
dspark_cfg = {
"target_layer_ids": list(config.dspark_target_layer_ids),
"block_size": config.dspark_block_size,
"mask_token_id": config.dspark_mask_token_id,
"enable_confidence_head": config.dspark_enable_confidence_head,
"confidence_head_with_markov":
config.dspark_confidence_head_with_markov,
"markov_head_type": config.dspark_markov_head_type,
"markov_rank": config.dspark_markov_rank,
"causal_head": bool(config.dspark_causal_proposal),
"contiguous_query_swa": bool(config.sliding_window_size > 0),
"sample_from_anchor": config.dspark_sample_from_anchor,
"heads_file": "dspark_heads.safetensors",
"heads_info_file": "dspark_heads_info.json",
}
out.update({
"draft_vocab_size":
config.vocab_size,
"base_model_hidden_size":
len(config.dspark_target_layer_ids) * config.hidden_size,
"dspark_config":
dspark_cfg,
})
if config.dspark_base:
out.update({
"dspark_config": {
"target_layer_ids": list(config.dspark_target_layer_ids),
"block_size": config.dspark_block_size,
"mask_token_id": config.dspark_mask_token_id,
"enable_confidence_head": config.dspark_enable_confidence_head,
"confidence_head_with_markov":
config.dspark_confidence_head_with_markov,
"markov_head_type": config.dspark_markov_head_type,
"markov_rank": config.dspark_markov_rank,
"sample_from_anchor": config.dspark_sample_from_anchor,
},
})
if config.eagle_base:
# EAGLE3 base: record which layers provide hidden states to the draft.
target_layers = list(config.eagle3_target_layer_ids)
if not target_layers:
n_layers = config.num_hidden_layers
target_layers = [2, n_layers // 2, n_layers - 3]
out["eagle_hidden_state_layers"] = target_layers
if config.reduced_vocab_size:
out["reduced_vocab_size"] = config.reduced_vocab_size
# KV cache dtype is baked in at export time. The C++ runtime parses it
# strictly from config.json (no engine-introspection back-patching).
# Mirrors llm_export.py: "fp8" when KV cache is quantised, otherwise "fp16".
out["kv_cache_dtype"] = ("fp8" if config.quant.kv_cache_quant == "fp8" else
"fp16")
# Hybrid models (Mamba / GDN / Nemotron-H) bake in recurrent-state and
# conv-state dtypes at export time. The authoritative source is the
# model class itself — `export_onnx` constructs dummy tensors with these
# dtypes, which in turn fix the ONNX binding dtypes the engine is built
# with. Reading them from the same class attribute used there guarantees
# the config string cannot drift from the engine binding. The C++ runtime
# validator cross-checks the config dtype against the engine binding at
# init, so a drift would fail loudly at load time; this keeps both sides
# pinned to one source.
if out.get("num_linear_attn_layers", 0) > 0:
for attr, key in (("RECURRENT_STATE_DTYPE", "recurrent_state_dtype"),
("CONV_STATE_DTYPE", "conv_state_dtype")):
torch_dtype = getattr(model, attr, None)
if torch_dtype is None:
raise AttributeError(
f"{type(model).__name__} is hybrid (num_linear_attn_layers>0) "
f"but does not expose {attr}. Add a class-level {attr} "
f"(torch.dtype) to the model class; its value must match "
f"the dtype of the dummy state tensor its export_onnx "
f"builds and the dtype mandated by the plugin schema.")
out[key] = _torch_dtype_to_config_str(torch_dtype)
if world_size > 1 or tp_size > 1:
# Rank-local fields are emitted throughout this function. Capture
# them only after the complete runtime config has been assembled,
# then restore the shared top-level model view.
override_fields = list(_RANK_LOCAL_OVERRIDE_FIELDS)
if config.gdn_cfg is not None:
override_fields.extend(("recurrent_state_num_heads", "conv_dim"))
local_shape_values = {
field: out[field]
for field in override_fields if field in out
}
if "kv_layer_configs" in out:
local_shape_values["kv_layer_configs"] = [
dict(layer) if layer is not None else None
for layer in out["kv_layer_configs"]
]
# Only TP exports need a shared world-level view. Single-device and
# specialized exports (for example reduced-layer validation, MTP
# draft, and TTS code-predictor models) must retain model.config
# because it describes the ONNX graph that was actually exported.
_apply_global_model_fields(out, global_llm_config)
if isinstance(global_llm_config, dict):
if config.gdn_cfg is not None:
for field in ("recurrent_state_num_heads", "conv_dim"):
if field in local_shape_values:
out[field] = local_shape_values[field] * tp_size
if "kv_layer_configs" in local_shape_values:
out["kv_layer_configs"] = _tp_global_kv_layer_configs(
local_shape_values["kv_layer_configs"], tp_size)
config_overrides: Dict[str, Any] = {}
for field, local_value in local_shape_values.items():
if out.get(field) != local_value:
config_overrides[field] = local_value
rank_config: Dict[str, Any] = {"rank": rank}
if tp_rank != rank:
rank_config["tp_rank"] = tp_rank
if config_overrides:
rank_config["config_overrides"] = config_overrides
out["rank_configs"] = _merge_rank_config(existing_rank_configs,
rank_config)
return out
def _build_alpamayo_tokenizer(config: Dict[str, Any], out_dir: str) -> None:
"""Build and save the Alpamayo-R1 tokenizer with added trajectory tokens.
The base tokenizer comes from the VLM (e.g. Qwen3-VL-8B-Instruct).
Alpamayo adds discrete trajectory tokens (<i0> .. <i767>) and special
trajectory tokens (<|traj_history|>, <|traj_future|>, etc.) on top.
"""
vlm_name = config.get("vlm_name_or_path", "Qwen/Qwen3-VL-8B-Instruct")
if not vlm_name:
return
try:
from transformers import AutoTokenizer
try:
tokenizer = AutoTokenizer.from_pretrained(vlm_name,
trust_remote_code=True)
except (OSError, ValueError) as online_exc:
logger.warning(
"Failed to load Alpamayo tokenizer from %s (%s); "
"retrying local cache", vlm_name, online_exc)
tokenizer = AutoTokenizer.from_pretrained(vlm_name,
trust_remote_code=True,
local_files_only=True)
# Add discrete trajectory tokens
traj_vocab_size = config.get("traj_vocab_size", 768)
if traj_vocab_size:
discrete_tokens = [f"<i{v}>" for v in range(traj_vocab_size)]
tokenizer.add_tokens(discrete_tokens)
# Add special trajectory tokens
_TRAJ_TOKENS = [
"<|traj_history|>",
"<|traj_future|>",
"<|traj_history_start|>",
"<|traj_future_start|>",
"<|traj_history_end|>",
"<|traj_future_end|>",
]
add_special = config.get("add_special_tokens", False)
if add_special:
_SPECIAL_TOKENS_KEYS = [
"prompt_start",
"prompt_end",
"image_start",
"image_pre_tkn",
"image_end",
"traj_history_start",
"traj_history_pre_tkn",
"traj_history_end",
"cot_start",
"cot_end",
"meta_action_start",
"meta_action_end",
"traj_future_start",
"traj_future_pre_tkn",
"traj_future_end",
"traj_history",
"traj_future",
"image_pad",
"vectorized_wm",
"vectorized_wm_start",
"vectorized_wm_end",
"vectorized_wm_pre_tkn",
"route_start",
"route_pad",
"route_end",
"question_start",
"question_end",
"answer_start",
"answer_end",
]
special_tokens = ["<|" + k + "|>" for k in _SPECIAL_TOKENS_KEYS]
tokenizer.add_tokens(special_tokens, special_tokens=True)
else:
tokenizer.add_tokens(_TRAJ_TOKENS, special_tokens=True)
os.makedirs(out_dir, exist_ok=True)
tokenizer.save_pretrained(out_dir)
logger.info("Saved Alpamayo tokenizer (%d tokens) to %s",
len(tokenizer), out_dir)
except (ImportError, OSError, ValueError) as exc:
logger.warning("Failed to build Alpamayo tokenizer: %s", exc)
def _runtime_embedding_scale(model: "CausalLM") -> float:
"""Return the scale folded into runtime token embedding sidecars."""
return runtime_embedding_scale(model.config)
def runtime_embedding_scale(config) -> float:
"""Return the scale folded into the runtime token embedding table of a model with *config*."""
explicit_scale = getattr(config, "embedding_scale", None)
if explicit_scale is not None:
return float(explicit_scale)
if str(getattr(config, "model_type", "")).startswith("gemma4"):
return math.sqrt(float(config.hidden_size))
return 1.0
def _gemma_ple_checkpoint_metadata(model_dir, config):
from . import ple_source
location = None
weight_scale = None
if (os.path.isfile(os.path.join(model_dir, "model.safetensors.index.json"))
or any(
name.endswith(".safetensors")
for name in os.listdir(model_dir))):
key, identity = ple_source.gemma_ple_identity(
model_dir, config.is_diffusion_gemma)
if identity["dtype"] == "F8_E4M3":
weight_scale = ple_source.gemma_ple_weight_scale(model_dir, key)
else:
import torch
index_path = os.path.join(model_dir, "pytorch_model.bin.index.json")
if os.path.isfile(index_path):
with open(index_path) as source:
weight_map = json.load(source)["weight_map"]
key = ple_source.select_gemma_ple_tensor(weight_map,
config.is_diffusion_gemma)
filename = weight_map[key]
path = ple_source.checkpoint_shard_path(model_dir, filename)
state = torch.load(path, map_location="meta", weights_only=True)
else:
filename = "pytorch_model.bin"
path = ple_source.checkpoint_shard_path(model_dir, filename)
state = torch.load(path, map_location="meta", weights_only=True)
state = state.get("state_dict", state)
key = ple_source.select_gemma_ple_tensor(state,
config.is_diffusion_gemma)
state = state.get("state_dict", state)
tensor = state[key]
offset = getattr(tensor.untyped_storage(), "_checkpoint_offset", None)
if offset is None or not tensor.is_contiguous():
raise ValueError(
"PLE checkpoint requires a contiguous tensor with an archive byte offset"
)
offset += tensor.storage_offset() * tensor.element_size()
byte_count = tensor.numel() * tensor.element_size()
dtype = {
torch.float16: "F16",
torch.bfloat16: "BF16",
torch.float32: "F32"
}.get(tensor.dtype)
identity = {
"dtype": dtype,
"shape": list(tensor.shape),
"bytes": byte_count,
"samples": ple_source.read_ple_samples(path, offset, byte_count)
}
location = {"file": filename, "offset": offset, "bytes": byte_count}
return ple_source.make_gemma_ple_source_metadata(
key, identity, config.vocab_size_per_layer_input,
config.num_hidden_layers * config.hidden_size_per_layer_input,
config.hidden_size_per_layer_input, location, weight_scale)
def write_runtime_artifacts(model: "CausalLM",
model_dir: str,
out_dir: str,
fp8_embedding: bool = False,
reduced_vocab_dir: str = "",
config_filename: str = "config.json",
write_shared_artifacts: bool = True) -> None:
"""Write runtime config and the shared model-input artifacts.
``config_filename`` selects the filename for the runtime config. Use
the default ``"config.json"`` for single-device exports, or
``"config_world{N}.json"`` for multi-rank exports.
``write_shared_artifacts`` controls whether embedding and tokenizer
sidecar files are emitted. TP ranks share those artifacts, so rank 0 can
write them once while every rank still writes the shared runtime config.
"""
import torch
from tensorrt_edgellm._safetensors_io import save_file
from ..chat_template import write_chat_template
os.makedirs(out_dir, exist_ok=True)
cfg_path = os.path.join(out_dir, config_filename)
existing_rank_configs = []
if os.path.exists(cfg_path):
with open(cfg_path) as f:
existing_cfg = json.load(f)
existing_rank_configs = existing_cfg.get("rank_configs", [])
root_cfg = {}
global_llm_config = None
if model_dir:
try:
root_cfg, global_llm_config = load_checkpoint_config_dicts(
model_dir)
except (OSError, ValueError, RuntimeError, ImportError) as exc:
logger.warning(
"Failed to load global checkpoint config from %s: %s",
model_dir, exc)
hf_cfg_path = os.path.join(model_dir, "config.json")
if os.path.exists(hf_cfg_path):
with open(hf_cfg_path) as _f:
root_cfg = json.load(_f)
cfg_json = build_runtime_llm_config_dict(
model,
global_llm_config=global_llm_config,
existing_rank_configs=existing_rank_configs)
kv_cache_scales = _alpamayo_kv_cache_scales(model, root_cfg, cfg_json)
if kv_cache_scales is not None:
cfg_json["kv_cache_scales"] = kv_cache_scales
# For VLM models, the C++ VLM runner (qwenViTRunner, internViTRunner)
# reads vision_config from the LLM config.json. Preserve it from the
# original HF config so the runtime can find deepstack_visual_indexes,
# num_position_embeddings, etc.
if root_cfg.get("vision_config"):
cfg_json["vision_config"] = root_cfg["vision_config"]
# Propagate eos_token_id so the C++ runtime can stop on any EOS
# token (e.g. Gemma4 uses [1, 106]). Union config.json with
# generation_config.json: HF generate stops on the generation_config
# set, which may extend the model config's single EOS (e.g. HunYuan
# adds <|extra_5|> alongside <|eos|>), and some models only set
# eos_token_id in one of the two files.
def _eos_ids(value) -> "list[int]":
if isinstance(value, list):
return [int(x) for x in value]
if isinstance(value, int):
return [value]
return []
eos_ids = _eos_ids(root_cfg.get("eos_token_id"))
if model_dir:
gen_cfg_path = os.path.join(model_dir, "generation_config.json")
if os.path.exists(gen_cfg_path):
with open(gen_cfg_path) as _gf:
gen_cfg = json.load(_gf)
eos_ids += _eos_ids(gen_cfg.get("eos_token_id"))
if eos_ids:
cfg_json["eos_token_id"] = list(dict.fromkeys(eos_ids))
ple_dir = getattr(model, "_ple_checkpoint_dir", model_dir)
if model.config.ple_enabled:
cfg_json["ple_pool"] = _gemma_ple_checkpoint_metadata(
ple_dir, model.config)
cfg_json["checkpoint_dir"] = os.path.abspath(ple_dir)
elif model.config.ngram_layers:
from . import ple_source
cfg_json["ple_pool"] = ple_source.make_ngram_pool_metadata(
ple_dir, model.config.ple_embed_dim, model.config.ngram_heads)
cfg_json["checkpoint_dir"] = os.path.abspath(ple_dir)
with open(cfg_path + ".tmp", "w") as output:
json.dump(cfg_json, output, indent=2)
os.replace(cfg_path + ".tmp", cfg_path)
logger.info("Wrote %s to %s", config_filename, out_dir)
if not write_shared_artifacts:
return
engine_role = _determine_engine_role(model.config)
# Every speculative draft shares the base model's embedding table.
if engine_role == "draft":
kind = _determine_spec_decode_type(model.config)
logger.info(
"%s draft: skipping embedding.safetensors (uses base model embedding)",
kind)
else:
embed = getattr(model, "embed_tokens", None)
if embed is None:
embed = getattr(getattr(model, "model", None), "embed_tokens",
None)
if embed is None:
embed = getattr(getattr(model, "backbone", None), "embeddings",
None)
if embed is not None:
embedding_path = os.path.join(out_dir, "embedding.safetensors")
from .embedding_quantization import write_modelopt_embedding
runtime_weight = getattr(embed, "runtime_weight", None)
quantized_weight = (not callable(runtime_weight)
and embed.weight.dtype
in (torch.float8_e4m3fn, torch.uint8))
if quantized_weight:
if fp8_embedding:
raise ValueError(
"--fp8-embedding cannot requantize a ModelOpt embedding"
)
if not model_dir or not write_modelopt_embedding(
model_dir, embed, _runtime_embedding_scale(model),
embedding_path):
raise ValueError(
"Quantized token embedding requires ModelOpt scales in the checkpoint"
)
else:
weight = (runtime_weight() if callable(runtime_weight) else
embed.weight.data).detach().cpu()
embedding_scale = _runtime_embedding_scale(model)
if embedding_scale != 1.0 and not fp8_embedding:
weight = weight * embedding_scale
# C++ runtime requires FP16 (or FP8) embedding; cast if needed.
if weight.dtype in (torch.float32, torch.bfloat16):
weight = weight.to(torch.float16)
if fp8_embedding:
from .embedding_quantization import \
quantize_embedding_to_fp8
embedding_fp8, scales = quantize_embedding_to_fp8(weight)
save_file(
{
"embedding": embedding_fp8,
"embedding_scale": scales * embedding_scale
}, embedding_path)
logger.info("Wrote FP8 embedding.safetensors (%s)",
list(weight.shape))
else:
save_file({"embedding": weight}, embedding_path)
logger.info("Wrote embedding.safetensors (%s)",
list(weight.shape))
else:
logger.warning(
"embed_tokens not found; skipping embedding.safetensors")
# Alpamayo-R1: tokenizer lives in the VLM checkpoint, not in model_dir.
# Build it first so that tokenizer files exist before the copy loop
# (which is a no-op for Alpamayo) and before write_chat_template.
if root_cfg.get("model_type") == "alpamayo_r1":
_build_alpamayo_tokenizer(root_cfg, out_dir)
for fname in RUNTIME_TOKENIZER_FILENAMES:
src = os.path.join(model_dir, fname)
if os.path.exists(src):
shutil.copy2(src, os.path.join(out_dir, fname))
logger.info("Copied %s", fname)
# If tokenizer.json is missing but vocab.json+merges.txt exist (GPT-2
# format, used by Qwen3-ASR/TTS), generate tokenizer.json using the
# transformers library so the C++ runtime can load it.
tok_json_dst = os.path.join(out_dir, "tokenizer.json")
if not os.path.exists(tok_json_dst) and model_dir:
vocab_src = os.path.join(model_dir, "vocab.json")
merges_src = os.path.join(model_dir, "merges.txt")
if os.path.exists(vocab_src) and os.path.exists(merges_src):
try:
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(model_dir)
tok.save_pretrained(out_dir)
logger.info(
"Generated tokenizer.json from vocab.json+merges.txt")
except (OSError, ValueError, ImportError):
logger.warning("Failed to generate tokenizer.json; "
"copying vocab.json and merges.txt as fallback")
shutil.copy2(vocab_src, os.path.join(out_dir, "vocab.json"))
shutil.copy2(merges_src, os.path.join(out_dir, "merges.txt"))
# EAGLE3 draft: save d2t (draft-to-target vocab map)
d2t = getattr(model, "d2t", None)
if d2t is not None:
d2t_cpu = d2t.data.cpu().to(torch.int32)
save_file({"d2t": d2t_cpu}, os.path.join(out_dir, "d2t.safetensors"))
logger.info("Wrote d2t.safetensors (%s)", list(d2t_cpu.shape))
from ..vocab_reduction.onnx_export import copy_reduced_vocab_artifacts
copy_reduced_vocab_artifacts(model, out_dir, reduced_vocab_dir)
if engine_role != "draft":
write_chat_template(model_dir, out_dir)