# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Checkpoint-native Python API for TensorRT Edge-LLM.
Checkpoints are lowered directly by :mod:`tensorrt_edgellm.builder`. Weights
externalized by the builder are validated and loaded once when the runtime
starts; this module has no ONNX export/build path.
Example::
from tensorrt_edgellm.server import LLM, SamplingParams
llm = LLM(model="Qwen/Qwen3.5-0.8B")
outputs = llm.generate(
["What is the capital of France?"],
SamplingParams(temperature=0.7, max_tokens=256),
)
for output in outputs:
print(output.text)
# Or start an OpenAI-compatible server:
llm.serve(port=8000)
"""
import importlib
import json
import logging
import math
import os
import threading
import time
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterator, List,
Mapping, Optional, Sequence, Tuple, Union)
from ..config import (DEFAULT_MAX_QUEUED_REQUESTS, IFB_SCHEDULING_DEFAULTS,
ContextCacheConfig, PleRuntimeConfig, ServerConfigError,
validate_ifb_scheduling_options)
from ..parsing.tool_calling import (ToolConfig, parse_assistant_output,
validate_tool_request)
from .engine_layout import BundleLayout, EngineType, inspect_bundle
logger = logging.getLogger("edgellm.server")
if TYPE_CHECKING:
from .engine_build import BuildOptions
_PLUGIN_LIB_NAME = "libNvInfer_edgellm_plugin.so"
_MAX_LOGIT_BIAS_TOKENS = 1024
_MAX_LOGIT_BIAS_TOKEN_ID = (1 << 31) - 1
_MIN_LOGIT_BIAS = -100.0
_MAX_LOGIT_BIAS = 100.0
_DEFAULT_MAX_INPUT_LEN = 4096
_DEFAULT_MAX_BATCH_SIZE = 1
_DEFAULT_MAX_KV_CACHE_CAPACITY = 8192
_DEFAULT_DRAFT_TOP_K = 10
_DEFAULT_DRAFT_STEP = 6
_CONTEXT_CACHE_COUNTER_FIELDS = ("admitted_sequences", "hit_sequences",
"media_aware_sequences",
"lookup_bypass_sequences",
"forced_cold_sequences", "matched_tokens",
"reused_tokens", "committed_publications",
"existing_publications", "hybrid_restores",
"hybrid_snapshot_pressure_skips",
"current_records", "evicted_records")
_CONTEXT_CACHE_POOL_FIELDS = ("base_kv_pages", "recurrent_snapshots",
"partial_kv_snapshots", "swa_kv_pages")
_QWEN_VISUAL_MODEL_TYPES = ("qwen2_vl", "qwen2_5_vl", "qwen3_vl", "qwen3_5",
"qwen3_omni", "cosmos3_edge_vision")
def _visual_model_family(model_type: str) -> Optional[str]:
"""Server-side media family for a visual model_type; None if unsupported."""
# Audio-side Qwen3-Omni types share the qwen3_omni prefix with its visual
# encoder, so exclude them before matching the Qwen visual prefixes.
if any(tag in model_type for tag in ("audio", "code2wav", "asr")):
return None
if "internvl" in model_type:
return "internvl"
if model_type == "muse_glimmer_vision":
return "muse"
if "nemotron" in model_type:
return "nemotron"
if model_type.startswith(_QWEN_VISUAL_MODEL_TYPES):
return "qwen"
return None
# ---------------------------------------------------------------------------
# Public data classes
# ---------------------------------------------------------------------------
[docs]
@dataclass
class SamplingParams:
"""Sampling parameters for one generation request."""
temperature: float = 0.7
top_p: float = 0.9
top_k: int = 50
seed: Optional[int] = None
max_tokens: int = 2048
enable_thinking: bool = False
reasoning_effort: str = ""
disable_spec_decode: bool = False
ignore_eos: bool = False
num_logprobs: int = 0
# Lower is served first; honoured under in-flight batching with the
# priority scheduling policy.
priority: int = 0
stop: List[str] = field(default_factory=list)
logit_bias: Dict[int, float] = field(default_factory=dict)
# Low-level grammar constraint as (guide_type, guide_text); see
# _normalize_guided_decoding. None means unconstrained.
guided_decoding: Optional[Tuple[str, str]] = None
# Append grammar-forced tokens without forward passes; only with guided_decoding.
jump_forward: bool = False
skip_special_tokens: bool = True
reuse_context: bool = True
# None lets the deployment choose: false for a speculative request reusing
# context on a hybrid model (only the prompt is cached there), else true.
cache_generated_tokens: Optional[bool] = None
@dataclass
class LogprobEntry:
"""One top-K log-probability entry for a single generated token.
``token`` is the piece decoded as UTF-8 with ``errors="replace"`` (a
byte-level BPE token may be only part of a multi-byte character, so it can
contain U+FFFD); ``bytes`` carries the raw token bytes losslessly.
"""
token_id: int
logprob: float
token: str
bytes: List[int]
def _convert_logprobs(raw) -> List[List[LogprobEntry]]:
"""Convert the C++/nanobind logprobs (list of list of native LogprobEntry with
a raw-bytes ``piece``) into engine LogprobEntry dataclasses."""
return [[
LogprobEntry(token_id=e.token_id,
logprob=e.logprob,
token=e.piece.decode("utf-8", "replace"),
bytes=list(e.piece)) for e in step
] for step in raw]
[docs]
@dataclass
class CompletionOutput:
"""Output of a single generation request."""
text: str = ""
token_ids: List[int] = field(default_factory=list)
prompt_tokens: Optional[int] = None
#: Prompt tokens served from reused state rather than prefilled.
reused_prompt_tokens: Optional[int] = None
finish_reason: Optional[str] = None
logprobs: List[List[LogprobEntry]] = field(default_factory=list)
tool_calls: List[Dict[str, Any]] = field(default_factory=list)
reasoning: Optional[str] = None
[docs]
@dataclass
class GenerationMetrics:
"""Native throughput timings and client-observed latency in seconds.
Native counts and phase times are independent of stream consumption.
Incomplete native snapshots fall back to received token counts; unavailable
phase times remain zero.
TTFT and total_time include admission, rendering, and consumer overhead.
generated_tokens includes prefill output; decoded_tokens and generation_time
cover only work after prefill. Use decode_tokens_per_second for throughput.
"""
prompt_tokens: Optional[int] = None
processed_prompt_tokens: Optional[int] = None
prompt_time: float = 0.0
generated_tokens: int = 0
time_to_first_token: Optional[float] = None
generation_time: float = 0.0
total_time: float = 0.0
decoded_tokens: int = 0
@property
def decode_tokens_per_second(self) -> float:
"""Native decode throughput, or zero when decode timing is unavailable."""
if self.generation_time <= 0:
return 0.0
return self.decoded_tokens / self.generation_time
[docs]
@dataclass
class StreamDelta:
"""Single delta from a streaming generation.
Text deltas carry ``text``/``token_ids``; audio deltas (Omni streaming)
carry ``audio_bytes`` (int16 LE mono PCM) instead. ``finished`` marks the
end of the text stream; generator exhaustion ends the audio stream.
"""
text: str = ""
token_ids: List[int] = field(default_factory=list)
prompt_tokens: Optional[int] = None
reused_prompt_tokens: Optional[int] = None
finished: bool = False
finish_reason: Optional[str] = None
logprobs: List[List[LogprobEntry]] = field(default_factory=list)
audio_bytes: Optional[bytes] = None
[docs]
@dataclass
class AudioParams:
"""Talker / vocoder knobs for one Omni audio-output request."""
voice: str = ""
talker_temperature: float = 0.9
talker_top_k: int = 50
talker_top_p: float = 1.0
repetition_penalty: float = 1.05
max_audio_length: int = 4096
codec_chunk_frames: int = 10
talker_prefill_threshold: int = 4
#: Sample rate of Omni Code2Wav PCM output.
OMNI_AUDIO_SAMPLE_RATE = 24000
def _native_audio_params(rt, audio: "AudioParams"):
"""Convert the AudioParams dataclass to the nanobind OmniAudioParams."""
omni_params = rt.OmniAudioParams()
omni_params.speaker_name = audio.voice
for name, value in asdict(audio).items():
if name != "voice":
setattr(omni_params, name, value)
return omni_params
[docs]
class CancellableIterator:
"""Wrap a generator with cancellation that can interrupt native waits.
Args:
iterator: Generator to consume and close.
cancel: Callable that requests producer cancellation without waiting for
iteration to finish. It must tolerate repeated calls.
``cancel()`` signals the producer without closing an executing generator.
``close()`` signals cancellation once and closes the generator when it is
not executing in another thread. The caller must wait for active iteration
to finish, then close again, before releasing resources owned by that
generator.
"""
[docs]
def __init__(self, iterator, cancel) -> None:
self._iterator = iterator
self._cancel = cancel
self._cancel_lock = threading.Lock()
self._cancelled = False
self._active_reads = 0
def __iter__(self):
return self
def __next__(self):
with self._cancel_lock:
self._active_reads += 1
try:
return next(self._iterator)
finally:
with self._cancel_lock:
self._active_reads -= 1
[docs]
def cancel(self) -> None:
"""Request cancellation without closing an executing iterator."""
self._cancel()
[docs]
def close(self) -> None:
with self._cancel_lock:
cancel = not self._cancelled
self._cancelled = True
reading = self._active_reads > 0
if cancel:
# Cancellation can invoke a user callback that closes this stream.
self._cancel()
if reading:
# Closing a generator another thread is running can crash the
# interpreter. The cancel signal wakes that next(); the caller
# closes again once it returns.
return
try:
self._iterator.close()
except (RuntimeError, ValueError):
# The generator is still unwinding; the caller closes again.
pass
def _pump_channels(rt, run, text_channel, audio_channel, sem=None):
"""Drive one generation in a worker thread, yielding StreamDeltas.
Shared by the Omni dual-stream path (both channels) and the standalone
TTS path (``text_channel=None``). The drain-once retry after
is_finished()/is_cancelled() closes the race where the producer finishes
between an empty pop and the check. The worker releases ``sem`` when the
C++ call returns.
"""
def _cancel():
if text_channel is not None:
text_channel.cancel()
audio_channel.cancel()
def _iterate():
error_holder = [None]
def _run():
try:
run()
except Exception as error: # noqa: BLE001 - re-raised below
error_holder[0] = error
_cancel()
finally:
if sem is not None:
sem.release()
worker = threading.Thread(target=_run, daemon=True)
worker.start()
text_done = text_channel is None
audio_done = False
try:
while not (text_done and audio_done):
if not text_done:
chunk = text_channel.wait_pop(timeout_ms=20)
if chunk is None and (text_channel.is_finished()
or text_channel.is_cancelled()):
chunk = text_channel.try_pop()
if chunk is None:
text_done = True
if chunk is not None:
reason = finish_reason_name(
rt, chunk.reason) if chunk.finished else None
reused = getattr(chunk, "reused_prompt_token_count",
-1)
yield StreamDelta(
text=chunk.text,
token_ids=list(chunk.token_ids),
prompt_tokens=(chunk.prompt_token_count
if chunk.prompt_token_count >= 0
else None),
reused_prompt_tokens=(reused
if reused >= 0 else None),
finished=chunk.finished,
finish_reason=reason,
logprobs=_convert_logprobs(chunk.logprobs),
)
text_done = chunk.finished
if not audio_done:
# Text drives pacing while it flows (non-blocking audio
# poll); once text ends, block on audio instead.
audio_chunk = audio_channel.wait_pop(
timeout_ms=100 if text_done else 0)
if audio_chunk is None and (audio_channel.is_finished() or
audio_channel.is_cancelled()):
audio_chunk = audio_channel.try_pop()
if audio_chunk is None:
audio_done = True
if audio_chunk is not None:
if audio_chunk.pcm16:
yield StreamDelta(audio_bytes=audio_chunk.pcm16)
audio_done = audio_chunk.is_final
finally:
if not (text_done and audio_done):
_cancel()
# Channels own buffers consumed by the native worker. Runtime
# teardown and admission release must wait for worker exit.
worker.join()
if error_holder[0] is not None:
raise error_holder[0]
return CancellableIterator(_iterate(), _cancel)
def _stream_tts(rt,
runtime,
text: str,
audio: "AudioParams",
sem,
infer_guard=None,
ensure_open=None) -> Iterator["StreamDelta"]:
"""Run one standalone TTS request; yields audio-only StreamDeltas.
``runtime`` is any nanobind object exposing ``handle_request_tts``
(LLMRuntime with the Omni stack loaded, or the TTS-only TTSRuntime).
``infer_guard`` serializes against text inference sharing the same CUDA
stream (unused by the TTS-only runtime, which serves no text).
"""
state = {}
def _cancel():
stream = state.get("stream")
if stream is not None:
stream.close()
def _iterate():
omni_params = _native_audio_params(rt, audio)
audio_channel = rt.AudioStreamChannel()
if sem is not None:
sem.acquire()
try:
if ensure_open is not None:
ensure_open()
except BaseException:
sem.release()
raise
def _run():
if infer_guard is None:
runtime.handle_request_tts(text, omni_params, audio_channel)
return
with infer_guard:
runtime.handle_request_tts(text, omni_params, audio_channel)
stream = _pump_channels(rt, _run, None, audio_channel, sem=sem)
state["stream"] = stream
yield from stream
return CancellableIterator(_iterate(), _cancel)
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
def _derive_model_id(model: str) -> str:
"""Return a clean id to advertise via /v1/models and echo in responses.
A local checkpoint path would otherwise leak the full filesystem path as
the model id, so use its directory name. A Hugging Face ID is kept as-is.
"""
src = model
if src and (os.path.isabs(src) or os.path.isdir(src)):
return os.path.basename(os.path.normpath(src))
return src
def _read_bundle_llm_config(bundle_dir: str) -> dict:
"""Read the text engine config (the base engine's for a speculative
bundle) from a complete model bundle."""
for filename in ("config.json", "base_config.json"):
cfg_path = os.path.join(bundle_dir, filename)
if os.path.exists(cfg_path):
with open(cfg_path) as f:
return json.load(f)
return {}
def resolve_cache_generated_tokens(params: "SamplingParams",
prefill_state_only: bool) -> bool:
"""Resolve one request's commit policy against the deployment.
A speculative request that looks up context on a hybrid base can publish
only the prompt endpoint: the native runtime refuses any other policy
there. Vanilla and lookup-bypass requests are not bound by it.
"""
value = params.cache_generated_tokens
endpoint_only = (prefill_state_only and params.reuse_context
and not params.disable_spec_decode)
if value is None:
return not endpoint_only
if not isinstance(value, bool):
raise ValueError("cache_generated_tokens must be boolean")
if value and endpoint_only:
raise ValueError(
"cache_generated_tokens=true is not supported with speculative "
"decoding on a hybrid model; omit it or set it to false")
return value
@dataclass(frozen=True)
class _SpecDecodeRuntimeOptions:
"""Drafting shape resolved from one speculative engine bundle."""
top_k: int
draft_step: Optional[int]
proposal_tokens: Optional[int]
verify_tokens: Optional[int]
def _internvl_geometry(cfg: dict, key: str) -> int:
"""InternVL writes vision_config image_size/patch_size as [H, W] and the
C++ builder and runtime both read index 0. Other families put a scalar
here and never read the result, so they get 0."""
value = (cfg.get("vision_config") or {}).get(key)
return int(value[0]) if isinstance(value, list) and value else 0
def _read_json(path: str) -> dict:
with open(path, encoding="utf-8") as file:
return json.load(file)
def _resolve_spec_decode_runtime_options(
bundle_dir: str,
method: str,
num_speculative_tokens: Optional[int],
draft_top_k: Optional[int],
draft_step: Optional[int],
proposal_tokens: Optional[int],
verify_tokens: Optional[int],
) -> _SpecDecodeRuntimeOptions:
"""Resolve method-specific defaults against the compiled engine profile."""
if method == "none":
return _SpecDecodeRuntimeOptions(
draft_top_k if draft_top_k is not None else _DEFAULT_DRAFT_TOP_K,
draft_step,
proposal_tokens,
verify_tokens,
)
base = _read_json(os.path.join(bundle_dir, "base_config.json"))
engine_method = str(base.get("spec_decode_type", method))
compatible_methods = {method}
if method == "mtp":
compatible_methods.add("gemma4_mtp")
if engine_method not in compatible_methods:
raise ValueError(
f"requested speculative method {method!r}, but the compiled "
f"bundle uses {engine_method!r}")
base_builder = base.get("builder_config", {})
if "max_spec_verify_tokens" not in base_builder:
raise ValueError(
"speculative base artifact is missing builder_config.max_spec_verify_tokens"
)
max_verify_tokens = int(base_builder["max_spec_verify_tokens"])
if engine_method in {"eagle3", "mtp", "gemma4_mtp"}:
if proposal_tokens is not None:
raise ValueError(
"proposal_tokens is not supported for iterative decoders; use draft_step"
)
top_k = (draft_top_k if draft_top_k is not None else
(_DEFAULT_DRAFT_TOP_K if engine_method == "eagle3" else 1))
resolved_step = (num_speculative_tokens
if num_speculative_tokens is not None else draft_step)
if resolved_step is None:
resolved_step = _DEFAULT_DRAFT_STEP
expected_verify = resolved_step + 1
if top_k == 1 and (verify_tokens is not None
and verify_tokens != expected_verify):
raise ValueError(
"chain iterative decoding requires verify_tokens=draft_step+1")
resolved_verify = (
verify_tokens if verify_tokens is not None else
expected_verify if top_k == 1 else max_verify_tokens)
return _SpecDecodeRuntimeOptions(top_k, resolved_step, None,
resolved_verify)
if engine_method in {"dflash", "jetspec", "domino", "dspark"}:
if draft_step is not None and draft_step != 1:
raise ValueError("block decoders only support draft_step=1")
return _SpecDecodeRuntimeOptions(
draft_top_k if draft_top_k is not None else 1,
1,
num_speculative_tokens
if num_speculative_tokens is not None else proposal_tokens,
verify_tokens,
)
raise ValueError(f"unsupported speculative engine mode {engine_method!r}")
def _ensure_plugin_path() -> None:
"""Set EDGELLM_PLUGIN_PATH if not already set.
Searches relative to this package and common build locations.
"""
if os.environ.get("EDGELLM_PLUGIN_PATH"):
return
project_root = Path(__file__).resolve().parents[3]
search_dirs = [
project_root / "build" / "core",
project_root / "build" / "lib",
]
for d in search_dirs:
candidate = d / _PLUGIN_LIB_NAME
if candidate.is_file():
os.environ["EDGELLM_PLUGIN_PATH"] = str(candidate)
return
def ifb_unsupported_reason(layout,
model_type: Optional[str] = None) -> Optional[str]:
"""Why ``--enable-in-flight-batching`` cannot serve this deployment, or
``None`` when it can.
The deployment-level half of the IFB support matrix, decided in one place
and once, at load time: the request engine drives one text runtime, so a
deployment whose runtime is reached beyond ``handleRequest`` (standalone
TTS) cannot honour the flag. A speculative deployment is served: its
draft engine travels into the request engine, and what the runtime
cannot seat into a running batch (bounded-SWA KV storage) the engine
refuses at construction with its own reason. A Qwen3-Omni
bundle is not refused: its Thinker is a text runtime the engine serves,
so the deployment runs text-only and the speech endpoints refuse per
request (``speech_available``). Request-level refusals (per-request LoRA,
speech output, trajectories) live in the engine's ``submit()``.
"""
if model_type == "qwen3_tts":
return ("standalone TTS models are not supported under in-flight "
"batching")
return None
_IFB_TEXT_ONLY_MESSAGE = (
"speech output is unavailable under in-flight batching: the engine serves "
"text only, so the Omni talker/code_predictor/code2wav engines were not "
"loaded. Start without --enable-in-flight-batching to use speech output.")
def _ifb_unsupported_error(reason: str) -> ValueError:
return ValueError(
"--enable-in-flight-batching is not supported for this deployment: "
f"{reason}. Start without the flag to use the blocking path.")
def _import_runtime():
"""Import the C++ nanobind module."""
try:
runtime_facade = importlib.import_module("tensorrt_edgellm.runtime")
native_package = importlib.import_module("tensorrt_edgellm._native")
try:
return runtime_facade.load()
except native_package.NativeManifestNotFoundError:
# Source checkouts have no generated variants.json.
pass
except ImportError:
pass
_ensure_plugin_path()
native_loader = importlib.import_module("tensorrt_edgellm._native.load")
for module_name in ("tensorrt_edgellm._edgellm_runtime",
"_edgellm_runtime"):
try:
module = importlib.import_module(module_name)
return native_loader.load_runtime_from_path(Path(module.__file__))
except ImportError:
pass
project_root = Path(__file__).resolve().parents[3]
search_dirs = []
if os.environ.get("EDGELLM_NANOBIND_DIR"):
search_dirs.append(Path(os.environ["EDGELLM_NANOBIND_DIR"]))
if os.environ.get("BUILD_DIR"):
search_dirs.append(Path(os.environ["BUILD_DIR"]) / "lib")
search_dirs.append(project_root / "build" / "lib")
search_dirs.extend(project_root.glob("build/lib.*"))
for cand_dir in search_dirs:
if not cand_dir.is_dir():
continue
so_files = list(cand_dir.glob("*_edgellm_runtime*.so"))
if so_files:
return native_loader.load_runtime_from_path(so_files[0])
raise ImportError(
"Could not import _edgellm_runtime. Build the C++ extension first:\n"
" cmake -S . -B build -DBUILD_PYTHON_BINDINGS=ON\n"
" cmake --build build --target _edgellm_runtime")
_GUIDE_TYPE_NAMES = ("json_object", "json_schema", "regex", "ebnf",
"structural_tag", "choice")
# The nanobind module is imported at runtime, so the enum members are resolved lazily.
_GUIDE_TYPE_ENUM = {
"json_object": lambda rt: rt.GuideType.JSON_OBJECT,
"json_schema": lambda rt: rt.GuideType.JSON_SCHEMA,
"regex": lambda rt: rt.GuideType.REGEX,
"ebnf": lambda rt: rt.GuideType.EBNF,
"structural_tag": lambda rt: rt.GuideType.STRUCTURAL_TAG,
"choice": lambda rt: rt.GuideType.CHOICE,
}
def _normalize_response_format(
response_format: Optional[Dict[str,
Any]]) -> Optional[Tuple[str, str]]:
"""Translate OpenAI's `response_format` into a low-level guide.
`strict` is ignored: guided decoding is always enforced, matching vLLM. The
high-level field cannot express regex / ebnf / structural_tag, which is why the
low-level `guided_decoding` field stays available alongside it.
"""
if response_format is None:
return None
if not isinstance(response_format, dict):
raise ValueError("'response_format' must be an object")
kind = response_format.get("type")
if kind in (None, "text"):
return None
if kind == "json_object":
return ("json_object", "")
if kind == "json_schema":
wrapper = response_format.get("json_schema")
if not isinstance(wrapper, dict):
raise ValueError("'response_format.json_schema' must be an object")
schema = wrapper.get("schema")
if not isinstance(schema, dict):
raise ValueError(
"'response_format.json_schema.schema' must be an object")
return ("json_schema", json.dumps(schema, sort_keys=True))
raise ValueError(
f"'response_format.type' must be text, json_object or json_schema, got {kind!r}"
)
def _normalize_guided_decoding(
guided_decoding: Optional[Dict[str,
Any]]) -> Optional[Tuple[str, str]]:
"""Validate the low-level `guided_decoding` field and flatten it to (type, guide).
Exactly one mode may be set. An empty string or list means "not requested" rather
than "constrain to nothing", which would otherwise compile to a grammar accepting
no token at all. `jump_forward` is validated here and read by
`_normalize_jump_forward`; it needs a mode to apply to.
"""
if guided_decoding is None:
return None
if not isinstance(guided_decoding, dict):
raise ValueError("'guided_decoding' must be an object")
unknown = set(guided_decoding) - set(_GUIDE_TYPE_NAMES) - {"jump_forward"}
if unknown:
raise ValueError("'guided_decoding' has unknown field(s): " +
", ".join(sorted(unknown)))
chosen: Optional[Tuple[str, str]] = None
for name in _GUIDE_TYPE_NAMES:
value = guided_decoding.get(name)
if value is None:
continue
if name == "json_object":
if not isinstance(value, bool):
raise ValueError(
"'guided_decoding.json_object' must be a boolean")
if not value:
continue
guide = ""
elif name == "choice":
# Checked before the string branch: a bare string would otherwise be carried
# through as a guide and only fail in the backend as "not valid JSON".
if not isinstance(value, list):
raise ValueError(
"'guided_decoding.choice' must be an array of strings")
if not value:
continue
guide = json.dumps(value)
elif isinstance(value, str):
if not value:
continue
guide = value
elif name in ("json_schema", "structural_tag") and isinstance(
value, dict):
guide = json.dumps(value, sort_keys=True)
else:
raise ValueError(f"'guided_decoding.{name}' must be a string")
if chosen is not None:
raise ValueError(
"'guided_decoding' sets more than one mode; exactly one is allowed"
)
chosen = (name, guide)
jump_forward = guided_decoding.get("jump_forward")
if jump_forward is not None:
if not isinstance(jump_forward, bool):
raise ValueError(
"'guided_decoding.jump_forward' must be a boolean")
if chosen is None:
raise ValueError(
"'guided_decoding.jump_forward' requires a guided_decoding mode"
)
return chosen
def _normalize_jump_forward(guided_decoding: Optional[Dict[str, Any]]) -> bool:
"""Read `guided_decoding.jump_forward`; unset means off.
Runs the same validation as `_normalize_guided_decoding`, so a malformed or
orphan flag is rejected whichever reader a caller uses.
"""
if guided_decoding is None:
return False
_normalize_guided_decoding(guided_decoding)
return bool(guided_decoding.get("jump_forward"))
def _normalize_logit_bias(
logit_bias: Optional[Dict[Any, Any]]) -> Dict[int, float]:
"""Validate and normalize an OpenAI-compatible logit_bias map."""
if logit_bias is None:
return {}
if not isinstance(logit_bias, dict):
raise ValueError(
"'logit_bias' must be an object mapping token IDs to bias values")
if len(logit_bias) > _MAX_LOGIT_BIAS_TOKENS:
raise ValueError(f"'logit_bias' has {len(logit_bias)} entries; max is "
f"{_MAX_LOGIT_BIAS_TOKENS}")
normalized: Dict[int, float] = {}
for token, bias in logit_bias.items():
if isinstance(token, bool):
raise ValueError(
f"'logit_bias' token ID {token!r} is not an integer")
if isinstance(token, int):
token_id = token
elif isinstance(token, str):
try:
token_id = int(token)
except ValueError as exc:
raise ValueError(
f"'logit_bias' token ID {token!r} is not an integer"
) from exc
else:
raise ValueError(
f"'logit_bias' token ID {token!r} is not an integer")
if token_id < 0 or token_id > _MAX_LOGIT_BIAS_TOKEN_ID:
raise ValueError(
f"'logit_bias' token ID must be in "
f"[0, {_MAX_LOGIT_BIAS_TOKEN_ID}], got {token_id}")
if isinstance(bias, bool) or not isinstance(bias, (int, float)):
raise ValueError(
f"'logit_bias' value for token ID {token_id} must be a number")
try:
bias_value = float(bias)
except OverflowError as exc:
raise ValueError(
f"'logit_bias' value for token ID {token_id} must be in "
f"[{_MIN_LOGIT_BIAS}, {_MAX_LOGIT_BIAS}], got {bias}") from exc
if (not math.isfinite(bias_value) or bias_value < _MIN_LOGIT_BIAS
or bias_value > _MAX_LOGIT_BIAS):
raise ValueError(
f"'logit_bias' value for token ID {token_id} must be in "
f"[{_MIN_LOGIT_BIAS}, {_MAX_LOGIT_BIAS}], got {bias_value}")
normalized[token_id] = bias_value
return normalized
def _native_context_cache_config(rt, config: ContextCacheConfig):
"""Translate validated public configuration to the nanobind value type."""
native = rt.ContextCacheConfig()
native.enabled = config.enabled
native.max_records = config.max_records
native.max_snapshots = config.max_snapshots
return native
def _native_ple_config(rt, config: PleRuntimeConfig):
native = rt.PleRuntimeConfig()
native.residency = (rt.PleResidency.CACHED if config.residency == "cached"
else rt.PleResidency.RESIDENT)
native.cache_slots = config.cache_slots
native.file_path = config.file_path
return native
def _set_context_cache_request_policies(
rt,
request,
params: SamplingParams,
prefill_state_only: bool = False) -> None:
"""Map the two public request controls onto native cache policies."""
if not isinstance(params.reuse_context, bool):
raise ValueError("reuse_context must be boolean")
request.context_cache_lookup_policy = (
rt.ContextCacheLookupPolicy.USE_CACHE
if params.reuse_context else rt.ContextCacheLookupPolicy.BYPASS)
request.context_cache_commit_policy = (
rt.ContextCacheCommitPolicy.INCLUDING_GENERATED_TOKENS
if resolve_cache_generated_tokens(params, prefill_state_only) else
rt.ContextCacheCommitPolicy.PREFILL_STATE_ONLY)
def _engine_config_value(builder_config: dict, field_name: str,
requested_value: int) -> int:
if field_name not in builder_config:
return requested_value
engine_value = int(builder_config[field_name])
if engine_value != requested_value:
logger.warning(
"Using %s=%d from engine builder_config instead of requested %d",
field_name,
engine_value,
requested_value,
)
return engine_value
# ---------------------------------------------------------------------------
# LLM class
# ---------------------------------------------------------------------------
[docs]
class LLM:
"""Checkpoint-direct entry point for offline and HTTP inference.
``model`` accepts a local checkpoint or Hugging Face ID. The checkpoint-direct
builder compiles every model component into a profile-specific cache bundle
and uses the builder's default plugin-weight policy.
"""
#: Selects the HTTP contract without a hierarchy of capability flags.
runtime_kind = "chat"
[docs]
def __init__(
self,
model: str,
*,
cache_dir: str = "",
engine_cache_max_size_gb: float = 50.0,
clear_engine_cache: bool = False,
max_input_len: int = _DEFAULT_MAX_INPUT_LEN,
max_batch_size: int = _DEFAULT_MAX_BATCH_SIZE,
max_kv_cache_capacity: int = _DEFAULT_MAX_KV_CACHE_CAPACITY,
max_spec_verify_tokens: Optional[int] = None,
max_spec_draft_tokens: Optional[int] = None,
max_image_tokens: Optional[int] = None,
max_image_tokens_per_image: Optional[int] = None,
draft_top_k: Optional[int] = None,
draft_step: Optional[int] = None,
proposal_tokens: Optional[int] = None,
verify_tokens: Optional[int] = None,
build_options: Optional["BuildOptions"] = None,
speculative_config: Optional[Any] = None,
context_cache_config: Optional[Union[ContextCacheConfig,
Mapping[str, Any]]] = None,
ple_config: Optional[Union[PleRuntimeConfig, Mapping[str,
Any]]] = None,
enable_in_flight_batching: bool = False,
ifb_scheduling_mode: str = "auto",
max_num_tokens: int = 0,
max_chunk_tokens: int = 0,
scheduling_policy: str = "fcfs",
preemption: str = "none",
priority_reserved_slots: int = 0,
max_pause_steps: int = 0,
on_progress: Optional[Callable[[str], None]] = None,
):
if not model:
raise ValueError("'model' must be provided")
if (isinstance(engine_cache_max_size_gb, bool)
or not math.isfinite(engine_cache_max_size_gb)
or engine_cache_max_size_gb <= 0):
raise ValueError("engine_cache_max_size_gb must be positive")
for name, value in (("draft_top_k", draft_top_k),
("draft_step", draft_step), ("proposal_tokens",
proposal_tokens),
("verify_tokens", verify_tokens),
("max_spec_verify_tokens", max_spec_verify_tokens),
("max_spec_draft_tokens", max_spec_draft_tokens)):
if value is None:
continue
if (isinstance(value, bool) or not isinstance(value, int)
or value <= 0):
raise ValueError(f"{name} must be a positive integer")
self._model_id = _derive_model_id(model)
self._draft_top_k = (draft_top_k if draft_top_k is not None else
_DEFAULT_DRAFT_TOP_K)
self._draft_step = draft_step
self._proposal_tokens = proposal_tokens
self._verify_tokens = verify_tokens
self._max_input_len = max_input_len
self._max_batch_size = max_batch_size
self._max_kv_cache_capacity = max_kv_cache_capacity
self._context_cache_config = ContextCacheConfig.parse(
context_cache_config)
self._ple_config = PleRuntimeConfig.parse(ple_config)
self._enable_in_flight_batching = enable_in_flight_batching
self._ifb_scheduling = {
"ifb_scheduling_mode": ifb_scheduling_mode,
"max_num_tokens": max_num_tokens,
"max_chunk_tokens": max_chunk_tokens,
"scheduling_policy": scheduling_policy,
"preemption": preemption,
"priority_reserved_slots": priority_reserved_slots,
"max_pause_steps": max_pause_steps,
}
self._admission_sem = threading.Semaphore(1)
self._infer_lock = threading.Lock()
self._close_lock = threading.Lock()
# Context-reuse observability: the counters as of the last logged
# request, so each request can report its own delta. Guarded because the
# streaming path calls the C++ runtime from a background thread.
self._ctx_reuse_metric_lock = threading.Lock()
self._prev_ctx_reused_tokens = 0
self._prev_ctx_matched_tokens = 0
self._prev_ctx_hit_sequences = 0
self._prev_ctx_admitted_sequences = 0
self._closed = False
self._runtime = None
self._engine = None
from .engine_build import BuildOptions, cache_root, prepare_model
options = build_options or BuildOptions(
max_input_len=max_input_len,
max_batch_size=max_batch_size,
max_kv_cache_capacity=max_kv_cache_capacity,
max_spec_verify_tokens=max_spec_verify_tokens,
max_spec_draft_tokens=max_spec_draft_tokens,
max_image_tokens=max_image_tokens,
max_image_tokens_per_image=max_image_tokens_per_image,
)
# Before the build, which can take minutes, against the batch size the
# engine is built with (build_options when given).
validate_ifb_scheduling_options(enable_in_flight_batching,
self._ifb_scheduling,
options.max_batch_size)
spec_method = options.spec_type
num_speculative_tokens = None
if speculative_config:
from ..config import SpeculativeConfig
spec = SpeculativeConfig.parse(speculative_config)
if spec.draft_top_k is not None:
draft_top_k = spec.draft_top_k
if spec.verify_tokens is not None:
verify_tokens = spec.verify_tokens
options = replace(options,
spec_type=spec.method,
draft_model_dir=spec.draft_model)
spec_method = spec.method
num_speculative_tokens = spec.num_speculative_tokens
# An explicit DFlash tree width sizes the base when no capacity is
# given (a DFlash2 base otherwise holds one block). The fanout alone
# never changes the artifact.
if (spec_method == "dflash" and options.max_spec_verify_tokens is None
and verify_tokens is not None):
options = replace(options, max_spec_verify_tokens=verify_tokens)
prepared = prepare_model(
model,
cache_dir,
options,
max_cache_size_bytes=int(engine_cache_max_size_gb * (1 << 30)),
clear_cache=clear_engine_cache,
on_progress=on_progress,
)
self._cache_dir = cache_root(cache_dir)
self._model_dir = prepared.model_dir
self._draft_model_dir = prepared.draft_model_dir
runtime_options = _resolve_spec_decode_runtime_options(
prepared.bundle_dir,
spec_method,
num_speculative_tokens,
draft_top_k,
draft_step,
proposal_tokens,
verify_tokens,
)
self._draft_top_k = runtime_options.top_k
self._draft_step = runtime_options.draft_step
self._proposal_tokens = runtime_options.proposal_tokens
self._verify_tokens = runtime_options.verify_tokens
self._init_from_bundle(prepared.bundle_dir)
if on_progress is not None:
on_progress("Loading runtime and capturing CUDA graphs")
self._load_runtime()
if on_progress is not None:
on_progress("Ready")
# ------------------------------------------------------------------
# Initialization
# ------------------------------------------------------------------
def _init_from_bundle(self, bundle_dir: str) -> None:
"""Validate one complete checkpoint-native runtime bundle."""
self._layout = inspect_bundle(bundle_dir)
if self._layout.engine_type not in (EngineType.LLM,
EngineType.SPEC_DECODE):
raise ValueError("no checkpoint-direct LLM engine found in "
f"{self._layout.root!r}")
self._bundle_dir = self._layout.root
llm_config = _read_bundle_llm_config(self._bundle_dir)
self._hybrid_base = bool(llm_config.get("num_linear_attn_layers"))
builder_config = llm_config.get("builder_config", {})
self._max_input_len = _engine_config_value(builder_config,
"max_input_len",
self._max_input_len)
self._max_batch_size = _engine_config_value(builder_config,
"max_batch_size",
self._max_batch_size)
self._max_kv_cache_capacity = _engine_config_value(
builder_config, "max_kv_cache_capacity",
self._max_kv_cache_capacity)
self._media_dir = self._layout.media_dir
logger.info("Using cached engine bundle: %s", self._bundle_dir)
def _load_runtime(self) -> None:
"""Load checkpoint-backed weights once, then initialize the runtime."""
self._rt = _import_runtime()
context_cache_config = _native_context_cache_config(
self._rt, self._context_cache_config)
logger.info("Loading runtime bundle from %s", self._bundle_dir)
# Asked for and not available is a configuration error, not a silent
# fallback: the operator would otherwise believe requests overlap when
# they are being served one at a time.
if getattr(self, "_enable_in_flight_batching", False):
reason = ifb_unsupported_reason(self._layout)
if reason:
raise _ifb_unsupported_error(reason)
ple_config = _native_ple_config(self._rt, self._ple_config)
speculative = self._layout.engine_type == EngineType.SPEC_DECODE
if speculative:
logger.info(
"Speculative decoding enabled (top_k=%d, draft_step=%s, "
"proposal_tokens=%s, verify_tokens=%s)",
self._draft_top_k,
self._draft_step,
self._proposal_tokens,
self._verify_tokens,
)
if speculative and not self._use_ifb():
self._runtime = self._rt.LLMRuntime(
engine_dir=self._bundle_dir,
multimodal_engine_dir=self._media_dir,
lora_weights_map={},
draft_top_k=self._draft_top_k,
draft_step=self._draft_step,
proposal_tokens=self._proposal_tokens,
verify_tokens=self._verify_tokens,
checkpoint_dir=self._model_dir,
draft_checkpoint_dir=self._draft_model_dir,
context_cache_config=context_cache_config,
ple_config=ple_config,
)
elif self._use_ifb():
# In-flight batching: the engine owns the runtime and the one
# thread driving it, so requests overlap at step boundaries
# instead of queueing on a Python lock. The deployment passed
# ifb_unsupported_reason above; the engine itself refuses what it
# cannot admit into. A speculative deployment hands its drafting
# shape to the engine, which seats each sequence's draft state as
# it admits it. An Omni bundle runs text-only here (see
# speech_available).
drafting = {}
if speculative:
drafting = dict(
draft_top_k=self._draft_top_k,
draft_step=self._draft_step,
proposal_tokens=self._proposal_tokens,
verify_tokens=self._verify_tokens,
draft_checkpoint_dir=self._draft_model_dir,
)
try:
self._engine = self._rt.RequestEngine(
self._bundle_dir,
self._media_dir,
{},
self._model_dir,
context_cache_config,
max_batch_size=self._max_batch_size,
ple_config=ple_config,
**self._ifb_scheduling,
**drafting,
)
policy = self._engine.resolved_policy
logger.info(
"In-flight batching enabled (max_batch_size=%d, "
"scheduling mode %s, %s execution)", self._max_batch_size,
policy.mode, policy.execution)
# The engine overlaps requests, so the gate no longer serialises
# them; it only bounds how many direct-API requests hold decoded
# media at once: the running batch plus the same queue depth the
# server allows (running + queued, as vLLM, SGLang and
# TensorRT-LLM count it). Callers past that block rather than
# being refused; the server path has its own gate and never
# takes this one.
self._admission_sem = threading.Semaphore(
self._max_batch_size + DEFAULT_MAX_QUEUED_REQUESTS)
if self._layout.has_speech:
logger.warning("Omni speech-output engines not loaded: %s",
_IFB_TEXT_ONLY_MESSAGE)
except ValueError as error:
# The scheduling options cannot be served by this deployment;
# the message names the option and the alternative.
raise ServerConfigError(str(error)) from error
except RuntimeError as error:
# The engine refused the deployment at construction (a batch it
# cannot reseat). The flag was explicit, so this is a startup
# error with the engine's own reason, not a quiet return to
# the blocking path.
raise _ifb_unsupported_error(str(error)) from error
if (getattr(self, "_engine", None) is None
and getattr(self, "_runtime", None) is None):
self._runtime = self._rt.LLMRuntime(
self._bundle_dir,
self._media_dir,
{},
self._model_dir,
context_cache_config,
ple_config=ple_config,
)
if getattr(self, "_runtime", None) is not None:
self._runtime.capture_decoding_cuda_graph()
self._load_omni_runtime()
logger.info("Engine loaded and ready.")
def _use_ifb(self) -> bool:
"""In-flight batching is opt-in: ``--enable-in-flight-batching``.
The flag is the whole decision: whether the deployment can honour it
was settled once in ``_load_runtime`` (``ifb_unsupported_reason``),
which raises rather than falling back. When enabled it serves text and
multimodal-input deployments; a media request joins the running
requests like any other — its encoders run at admission — except under
a visual-token pruner, where a media request runs alone because pruning
gates on an empty KV cache and a seated prefill would skip it.
"""
return bool(getattr(self, "_enable_in_flight_batching", False))
@property
def speech_available(self) -> bool:
"""Whether this process can produce speech: the Omni stack is in the
bundle and the runtime is the blocking one that drives it. Under
in-flight batching the engine owns the runtime and serves text only.
"""
return bool(self._layout.has_speech
and getattr(self, "_engine", None) is None)
def _load_omni_runtime(self) -> None:
"""Load the Qwen3-Omni audio-output stack when its engines exist."""
if not self._layout.has_speech:
return
if getattr(self, "_engine", None) is not None:
# Text-only mode: the Talker pipeline drives the runtime outside
# handleRequest, which the engine's actor does not mediate, so the
# speech engines stay unloaded and the speech endpoints refuse.
logger.warning("Omni speech-output engines not loaded: %s",
_IFB_TEXT_ONLY_MESSAGE)
return
logger.info("Auto-detected Omni engines: talker=%s",
self._layout.talker_dir)
self._runtime.load_omni(self._layout.talker_dir,
self._layout.code_predictor_dir,
self._layout.code2wav_dir, self._bundle_dir,
self._model_dir)
logger.info("Omni audio output ready.")
def _visual_config(self) -> dict:
"""Read the model-specific visual component configuration once."""
cached = getattr(self, "_visual_config_cache", None)
if cached is not None:
return cached
cfg: dict = {}
root = self._media_dir
cfg_path = os.path.join(root, "visual", "config.json")
if os.path.isfile(cfg_path):
try:
with open(cfg_path) as f:
cfg = json.load(f)
except (OSError, ValueError):
cfg = {}
self._visual_config_cache = cfg
return cfg
def _video_model_family(self) -> str:
"""Frame-sampling family ("qwen" / "internvl" / "muse" / "nemotron")
from the visual engine's model_type. Types without a video path (phi4mm,
gemma, ...) are rejected: their runners read only the first frame."""
cached = getattr(self, "_video_family_cache", None)
if cached is not None:
return cached
config = self._visual_config()
model_type = config.get("model_type", "")
root = self._media_dir
has_visual = os.path.isfile(
os.path.join(root, "visual", "visual.engine"))
family = _visual_model_family(model_type)
if not has_visual or family is None or (
family == "nemotron"
and not config.get("supports_video", True)):
# Covers audio-only engines (audio/ but no visual/) and model
# types whose runners have no video path (phi4mm, gemma, ...).
raise ValueError(
f"video input is not supported for model_type={model_type!r}"
" in this runtime bundle; supported families: Qwen-VL "
"(qwen2_vl/qwen2_5_vl/qwen3_vl/qwen3_5/qwen3_omni), "
"Cosmos3-Edge, InternVL, "
"Nemotron-Omni, and Muse-Glimmer")
self._video_family_cache = family
return family
def _video_frame_limits(self) -> dict:
"""Engine-profile inputs for frame-count clamping (see video_sampling):
builder token bounds from the visual config.json + patch geometry from
preprocessor_config.json. Empty dict when unavailable (no clamping)."""
cached = getattr(self, "_video_limits_cache", None)
if cached is not None:
return cached
limits: dict = {}
cfg = self._visual_config()
builder = cfg.get("builder_config") or {}
root = self._media_dir
pre: dict = {}
pre_path = os.path.join(root, "visual", "preprocessor_config.json")
if os.path.isfile(pre_path):
try:
with open(pre_path) as f:
pre = json.load(f)
except (OSError, ValueError):
pre = {}
pre = pre.get("image_processor", pre)
processor: dict = {}
processor_path = os.path.join(root, "visual", "processor_config.json")
if os.path.isfile(processor_path):
try:
with open(processor_path) as f:
processor = json.load(f)
except (OSError, ValueError):
processor = {}
video_processor = processor.get("video_processor") or {}
if builder.get("max_image_tokens"):
# Cosmos3 processes every sampled frame. Its processor omits the
# Qwen temporal-patching field, so match the compiled runner's
# default instead of applying Qwen's two-frame grouping.
temporal_patch_size = pre.get("temporal_patch_size")
if temporal_patch_size is None:
temporal_patch_size = (1 if cfg.get("model_type")
== "cosmos3_edge_vision" else 2)
limits = {
"model_type":
cfg.get("model_type", ""),
"min_image_tokens":
int(builder.get("min_image_tokens", 1)),
"max_image_tokens":
int(builder["max_image_tokens"]),
"max_image_tokens_per_image":
int(builder.get("max_image_tokens_per_image", 0)),
"max_cu_seqlen_groups":
int(builder.get("max_cu_seqlen_groups", 0)),
"patch_size":
int(pre.get("patch_size", 0)),
"merge_size":
int(pre.get("merge_size", 0)),
"temporal_patch_size":
int(temporal_patch_size),
"max_image_tokens_checkpoint":
int(pre.get("max_image_tokens", 0)),
"max_video_frame_tokens":
int(video_processor.get("max_video_frame_tokens", 0)),
# Nemotron-Omni video geometry (top-level visual config.json).
"video_pruning_rate":
float(cfg.get("video_pruning_rate", 0.0)),
"video_temporal_patch_size":
int(cfg.get("video_temporal_patch_size", 2)),
"video_target_num_patches":
int(cfg.get("video_target_num_patches", 1024)),
"downsample_ratio":
float(cfg.get("downsample_ratio", 0.5)),
# InternVL geometry (visual config.json vision_config).
"internvl_image_size":
_internvl_geometry(cfg, "image_size"),
"internvl_patch_size":
_internvl_geometry(cfg, "patch_size"),
}
self._video_limits_cache = limits
return limits
def _prepare_messages_for_runtime(
self,
messages: List[Dict[str, Any]],
):
"""Preserve structured messages for the model-owned C++ renderer."""
image_buffers = _load_image_buffers(self._rt, messages,
self._video_model_family,
self._video_frame_limits)
return _convert_messages_to_cpp(self._rt, messages), image_buffers
def _make_generation_request(
self,
messages: List[Dict[str, Any]],
params: SamplingParams,
*,
tools: Optional[Sequence[Dict[str, Any]]] = None,
tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
tool_config: Optional[ToolConfig] = None,
apply_chat_template: bool = True,
add_generation_prompt: bool = True,
stream_channel: Optional[Any] = None,
):
normalized_logit_bias = _normalize_logit_bias(params.logit_bias)
tool_config = tool_config or validate_tool_request(
messages, tools, tool_choice)
# The replay tail only matters for a prefill-state-only commit against a
# draft model with reuse enabled: that is the deployment whose
# checkpoint must land on a turn boundary the next render reproduces.
# A negative value asks the native renderer/tokenizer to derive it.
cache_config = getattr(self, "_context_cache_config", None)
prefill_state_only = self.publishes_prefill_state_only
derive_replay_tail = (
cache_config is not None and cache_config.enabled
and not resolve_cache_generated_tokens(params, prefill_state_only)
and self.has_draft_model and apply_chat_template
and add_generation_prompt)
replay_tail_length = -1 if derive_replay_tail else 0
cpp_messages, image_buffers = self._prepare_messages_for_runtime(
messages)
audio_buffers = _load_audio_buffers(self._rt, messages)
request = self._rt.LLMGenerationRequest()
req = self._rt.Request(messages=cpp_messages)
req.image_buffers = image_buffers
req.audio_buffers = audio_buffers
req.stop_strings = params.stop
req.sampling_seed = params.seed
req.logit_bias = normalized_logit_bias
if params.guided_decoding is not None:
guide_type, guide = params.guided_decoding
req.guided_decoding = self._rt.GuidedDecodingParams(
_GUIDE_TYPE_ENUM[guide_type](self._rt), guide,
params.jump_forward)
request.requests = [req]
if stream_channel is not None:
request.stream_channels = [stream_channel]
request.temperature = params.temperature
request.top_p = params.top_p
request.top_k = params.top_k
request.sampling_seed = params.seed
request.max_generate_length = params.max_tokens
request.apply_chat_template = apply_chat_template
request.add_generation_prompt = add_generation_prompt
request.enable_thinking = params.enable_thinking
request.reasoning_effort = params.reasoning_effort
request.tools = _convert_tools_to_cpp(self._rt, tool_config)
request.tool_choice = _convert_tool_choice_to_cpp(
self._rt, tool_config)
request.parallel_tool_calls = tool_config.parallel_tool_calls
request.disable_spec_decode = params.disable_spec_decode
request.skip_special_tokens = params.skip_special_tokens
request.ignore_eos = params.ignore_eos
request.num_logprobs = params.num_logprobs
request.priority = params.priority
_set_context_cache_request_policies(self._rt, request, params,
prefill_state_only)
request.context_cache_replay_tail_length = replay_tail_length
return request
def _make_completion_request(
self,
prompt: Union[str, List[int]],
params: SamplingParams,
):
messages = ([{
"role": "user",
"content": prompt
}] if isinstance(prompt, str) else [])
request = self._make_generation_request(
messages,
params,
apply_chat_template=False,
add_generation_prompt=False,
)
if isinstance(prompt, list):
request.pre_tokenized_input_ids = [prompt]
return request
def _count_prepared_prompt_tokens(self, request) -> Optional[int]:
"""Count tokens only for an explicit token-count API request."""
counter = getattr(self, "_engine", None) or self._runtime
if not hasattr(counter, "count_prompt_tokens"):
return None
rows = getattr(request, "requests", ())
if any(
getattr(row, "image_buffers",
()) or getattr(row, "audio_buffers", ())
or getattr(row, "past_trajectory", None) is not None
for row in rows):
return None
counts = counter.count_prompt_tokens(request)
return counts[0] if counts else None
def _parse_generation_output(
self,
text: str,
token_ids: List[int],
prompt_tokens: Optional[int],
finish_reason: Optional[str],
tool_config: ToolConfig,
*,
tool_parser: str = "auto",
reasoning_parser: str = "none",
) -> CompletionOutput:
parsed = parse_assistant_output(
text,
tool_config,
self._model_dir,
tool_parser=tool_parser,
reasoning_parser=reasoning_parser,
)
tool_calls = [call.to_openai() for call in parsed.tool_calls]
return CompletionOutput(
text=parsed.content,
token_ids=token_ids,
prompt_tokens=prompt_tokens,
finish_reason="tool_calls" if tool_calls else finish_reason,
tool_calls=tool_calls,
reasoning=parsed.reasoning or None,
)
def _complete_prepared_request(
self,
request,
params: SamplingParams,
tool_config: ToolConfig,
*,
tool_parser: str = "auto",
reasoning_parser: str = "none",
on_handle=None,
) -> CompletionOutput:
response = self._handle_request(request, on_handle=on_handle)
text = response.output_texts[0] if response.output_texts else ""
token_ids = response.output_ids[0] if response.output_ids else []
prompt_tokens = (response.prompt_token_counts[0]
if response.prompt_token_counts else None)
finish_reason = (finish_reason_name(self._rt,
response.finish_reasons[0])
if response.finish_reasons else "stop")
output = self._parse_generation_output(
text,
token_ids,
prompt_tokens,
finish_reason,
tool_config,
tool_parser=tool_parser,
reasoning_parser=reasoning_parser,
)
if params.num_logprobs > 0 and response.logprobs:
output.logprobs = _convert_logprobs(response.logprobs[0])
reused_counts = getattr(response, "reused_prompt_token_counts", None)
if reused_counts:
output.reused_prompt_tokens = reused_counts[0]
return output
# ------------------------------------------------------------------
# Inference API
# ------------------------------------------------------------------
def _admission(self):
"""Per-instance gate from media decode through inference completion:
queued requests must not each pin decoded frames. Semaphore, not Lock --
streaming releases from the worker/SSE side. One slot on the blocking
path, batch plus queue depth under in-flight batching (see
``_load_runtime``)."""
return self._admission_sem
def _infer_guard(self):
"""Lock serializing every entry into the stateful C++ runtime."""
return self._infer_lock
def _ensure_open(self) -> None:
# getattr: callable on duck-typed non-LLM objects in the server tests.
if self._closed or (self._runtime is None
and getattr(self, "_engine", None) is None):
raise RuntimeError("Edge-LLM runtime is closed")
@staticmethod
def _ratio(num: int, denom: int) -> float:
return num / denom if denom > 0 else 0.0
def _log_context_reuse_metrics(self) -> None:
"""Emit an INFO line making context-cache reuse visible.
``get_context_cache_metrics()`` returns cumulative coordinator counters
for the whole server run, so the per-request figures are the delta since
the previous logged request. ContextCacheMetrics carries no total-prompt
-token field, so the headline hit rate is sequence-level
(hit_sequences / admitted_sequences); reused/matched token counts are
reported alongside to show token-level reuse is actually happening.
"""
cc = self._runtime.get_context_cache_metrics()
if cc is None:
return
with self._ctx_reuse_metric_lock:
cum_reused = int(cc.reused_tokens)
cum_matched = int(cc.matched_tokens)
cum_hit_seqs = int(cc.hit_sequences)
cum_admitted = int(cc.admitted_sequences)
req_reused = cum_reused - self._prev_ctx_reused_tokens
req_matched = cum_matched - self._prev_ctx_matched_tokens
req_hit_seqs = cum_hit_seqs - self._prev_ctx_hit_sequences
req_admitted = cum_admitted - self._prev_ctx_admitted_sequences
self._prev_ctx_reused_tokens = cum_reused
self._prev_ctx_matched_tokens = cum_matched
self._prev_ctx_hit_sequences = cum_hit_seqs
self._prev_ctx_admitted_sequences = cum_admitted
logger.info(
"[context-reuse] request: reused=%d matched=%d hitSeqs=%d/%d "
"hitRate=%.4f | cumulative: reused=%d matched=%d hitSeqs=%d/%d "
"hitRate=%.4f records=%d hybridRestores=%d "
"hybridSnapshotPressureSkips=%d evictedRecords=%d",
req_reused,
req_matched,
req_hit_seqs,
req_admitted,
self._ratio(req_hit_seqs, req_admitted),
cum_reused,
cum_matched,
cum_hit_seqs,
cum_admitted,
self._ratio(cum_hit_seqs, cum_admitted),
int(cc.current_records),
int(cc.hybrid_restores),
int(cc.hybrid_snapshot_pressure_skips),
int(cc.evicted_records),
)
def _handle_request(self, request, on_handle=None):
"""Entry to the C++ runtime: submitted to the engine's actor under
in-flight batching, serialized under a lock on the blocking path.
``on_handle`` receives the engine's RequestHandle as soon as the
request is submitted, so a caller that gives up on the result (a
disconnected client) can cancel it instead of leaving it decoding in
its batch seat until ``max_tokens``.
"""
engine = getattr(self, "_engine", None)
if engine is not None:
self._ensure_open()
handle = engine.submit(request)
if on_handle is not None:
on_handle(handle)
response = handle.get()
else:
with self._infer_guard():
self._ensure_open()
response = self._runtime.handle_request(request)
# Callable on duck-typed non-LLM objects in the server tests, which have
# no config to inherit a class default from.
cache_config = getattr(self, "_context_cache_config", None)
# Per-request deltas of cumulative counters only mean something when
# requests run one at a time; under in-flight batching the counters are
# on /health instead.
if (cache_config is not None and cache_config.enabled
and self._runtime is not None):
self._log_context_reuse_metrics()
return response
[docs]
def close(self) -> None:
"""Drain active work and release native engines and device memory."""
with self._close_lock:
if self._closed:
return
self._closed = True
if getattr(self, "_engine", None) is not None:
# DRAIN lets accepted requests finish; the join inside makes
# this the same "no work in flight afterwards" guarantee the
# lock pair gives the blocking path.
self._engine.shutdown(self._rt.ShutdownMode.DRAIN)
self._engine = None
with self._admission_sem:
with self._infer_lock:
self._runtime = None
def __enter__(self) -> "LLM":
self._ensure_open()
return self
def __exit__(self, _exc_type, _exc, _traceback) -> None:
self.close()
[docs]
def generate(
self,
prompts: Union[str, List[str], List[List[Dict[str, Any]]]],
sampling_params: Optional[SamplingParams] = None,
*,
tools: Optional[Sequence[Dict[str, Any]]] = None,
tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
tool_parser: str = "auto",
reasoning_parser: str = "none",
) -> List[CompletionOutput]:
"""Generate completions for the given prompts.
Args:
prompts: A single prompt string, a list of prompt strings, or
a list of OpenAI-style message lists.
sampling_params: Sampling configuration. Defaults to
``SamplingParams()``.
tools: Optional OpenAI-compatible tool definitions.
tool_choice: Optional OpenAI-compatible tool choice.
Returns:
List of ``CompletionOutput`` objects, one per prompt.
"""
params = sampling_params or SamplingParams()
if isinstance(prompts, str):
prompts = [prompts]
message_batches = []
for p in prompts:
if isinstance(p, str):
message_batches.append([{"role": "user", "content": p}])
elif isinstance(p, list):
message_batches.append(p)
else:
raise TypeError(f"Unsupported prompt type: {type(p)}")
outputs = []
for messages in message_batches:
tool_config = validate_tool_request(messages, tools, tool_choice)
with self._admission():
self._ensure_open()
request = self._make_generation_request(
messages,
params,
tools=tool_config.tools,
tool_choice=tool_config.tool_choice,
tool_config=tool_config,
)
output = self._complete_prepared_request(
request,
params,
tool_config,
tool_parser=tool_parser,
reasoning_parser=reasoning_parser,
)
outputs.append(output)
return outputs
[docs]
def chat(
self,
messages: List[Dict[str, Any]],
sampling_params: Optional[SamplingParams] = None,
*,
tools: Optional[Sequence[Dict[str, Any]]] = None,
tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
tool_parser: str = "auto",
reasoning_parser: str = "none",
) -> CompletionOutput:
"""Single-turn chat completion (convenience wrapper).
Args:
messages: OpenAI-style message list.
sampling_params: Sampling configuration.
tools: Optional OpenAI-compatible tool definitions.
tool_choice: Optional OpenAI-compatible tool choice.
Returns:
A single ``CompletionOutput``.
"""
return self.generate([messages],
sampling_params,
tools=tools,
tool_choice=tool_choice,
tool_parser=tool_parser,
reasoning_parser=reasoning_parser)[0]
[docs]
def generate_stream(
self,
messages: List[Dict[str, Any]],
sampling_params: Optional[SamplingParams] = None,
*,
tools: Optional[Sequence[Dict[str, Any]]] = None,
tool_choice: Optional[Union[str, Dict[str, Any]]] = None,
prebuilt_request: Optional[Any] = None,
on_metrics: Optional[Callable[[GenerationMetrics], None]] = None,
) -> Iterator[StreamDelta]:
"""Stream generation deltas for a single message list.
Yields ``StreamDelta`` objects as tokens are produced. Under
in-flight batching the request is submitted to the engine and this
thread reads the attached ``StreamChannel`` directly; on the blocking
path ``handleRequest`` runs in a background thread instead.
``on_metrics`` receives final native throughput and wall-clock timings
after the producer stops, including when the stream is closed.
Collection is disabled when ``on_metrics`` is omitted. Per-request
metrics do not enable aggregate counters or CUDA event profiling.
Cancellation or closure before iteration reports empty metrics once,
without starting a request.
``cancel()`` requests cancellation; ``close()`` waits for the blocking
worker, or the request engine outcome when metrics are requested.
"""
params = sampling_params or SamplingParams()
metrics = GenerationMetrics()
state = {"on_metrics": on_metrics}
def report_metrics():
# Claim the callback before invoking it; cancellation can race iteration.
callback = state.pop("on_metrics", None)
if callback is not None:
callback(metrics)
def _cancel():
# close() after a clean EOF must stay a no-op: the outcome is
# published at the step that evicts the sequence, and a cancel
# planted before that records a completed request as cancelled.
if state.get("finished_cleanly"):
return
state["cancelled_by_caller"] = True
handle = state.get("handle")
if handle is not None:
handle.cancel()
channel = state.get("channel")
if channel is not None:
channel.cancel()
if not state.get("started"):
report_metrics()
def _iterate():
state["started"] = True
if state.get("cancelled_by_caller"):
report_metrics()
return
started = time.perf_counter() if on_metrics is not None else None
received_tokens = 0
def collect_metrics(chunk, observed_at=None):
nonlocal received_tokens
if on_metrics is None:
return
if chunk.prompt_token_count >= 0:
metrics.prompt_tokens = chunk.prompt_token_count
if chunk.token_ids:
if (metrics.time_to_first_token is None
and observed_at is not None):
metrics.time_to_first_token = observed_at - started
received_tokens += len(chunk.token_ids)
metrics.generated_tokens = max(metrics.generated_tokens,
received_tokens)
native_metrics = getattr(chunk, "metrics", None)
if (native_metrics is not None
and native_metrics.processed_prompt_tokens >= 0):
metrics.processed_prompt_tokens = native_metrics.processed_prompt_tokens
metrics.prompt_time = native_metrics.prompt_seconds
# Some runtimes do not sample during prefill and leave
# their native generation snapshot uninitialized.
metrics.generated_tokens = max(
metrics.generated_tokens,
native_metrics.generated_tokens)
metrics.generation_time = native_metrics.generation_seconds
metrics.decoded_tokens = native_metrics.decoded_tokens
self._ensure_open()
channel = self._rt.StreamChannel.create(
collect_metrics=on_metrics is not None)
state["channel"] = channel
channel.set_skip_special_tokens(params.skip_special_tokens)
# The HTTP layer owns admission for a prebuilt request. Direct
# callers take the per-LLM gate here; under in-flight batching it
# is wide enough for a batch plus a queue, and there is no worker
# thread: the engine queues the request and this thread just reads
# the channel it attached.
engine = getattr(self, "_engine", None)
sem = None if prebuilt_request is not None else self._admission()
if sem is not None:
sem.acquire()
try:
self._ensure_open()
if prebuilt_request is not None:
request = prebuilt_request
request.stream_channels = [channel]
else:
request = self._make_generation_request(
messages,
params,
tools=tools,
tool_choice=tool_choice,
stream_channel=channel,
)
except BaseException:
if sem is not None:
sem.release()
raise
error_holder = [None]
worker = None
if engine is not None:
state["handle"] = engine.submit(request)
if state.get("cancelled_by_caller"):
state["handle"].cancel()
else:
def _run():
try:
self._handle_request(request)
except Exception as exc:
error_holder[0] = exc
channel.cancel()
finally:
if sem is not None:
sem.release()
worker = threading.Thread(target=_run, daemon=True)
worker.start()
saw_finished = False
try:
while True:
chunk = channel.wait_pop(timeout_ms=200)
if state.get("cancelled_by_caller"):
if chunk is not None:
collect_metrics(chunk)
break
if chunk is None:
if channel.is_cancelled():
break
handle = state.get("handle")
if channel.is_finished() or (handle is not None
and handle.ready()):
chunk = channel.try_pop()
if chunk is None:
break
else:
continue
if on_metrics is not None:
collect_metrics(chunk, time.perf_counter())
reason = finish_reason_name(
self._rt, chunk.reason) if chunk.finished else None
if chunk.finished:
# Before the yield: a consumer that closes the stream
# right after the terminal delta must not be taken for
# a disconnect and cancel a request that completed.
state["finished_cleanly"] = True
reused = getattr(chunk, "reused_prompt_token_count", -1)
yield StreamDelta(
text=chunk.text,
token_ids=list(chunk.token_ids),
prompt_tokens=(chunk.prompt_token_count
if chunk.prompt_token_count >= 0 else
None),
reused_prompt_tokens=reused if reused >= 0 else None,
finished=chunk.finished,
finish_reason=reason,
logprobs=_convert_logprobs(chunk.logprobs),
)
if chunk.finished:
saw_finished = True
break
finally:
# The producer pushes the finished chunk before it marks the
# channel finished; a consumer that raced past that window
# must not cancel a stream that ended cleanly, or the tail
# below would mistake it for an actor failure and block on
# get() until the whole batch drains.
if not (saw_finished or channel.is_finished()
or channel.is_cancelled()):
channel.cancel()
if worker is not None:
worker.join()
else:
handle = state.get("handle")
if handle is not None and on_metrics is not None:
# Cancellation wakes get() before the actor publishes final metrics.
while not handle.ready():
time.sleep(0.01)
try:
handle.get()
except Exception as exc:
if not state.get("cancelled_by_caller"):
error_holder[0] = exc
if sem is not None:
sem.release()
# Final snapshots may arrive after cancellation; collect them
# without yielding text or inventing an observed first token.
while True:
chunk = channel.try_pop()
if chunk is None:
break
collect_metrics(chunk)
if on_metrics is not None:
metrics.total_time = time.perf_counter() - started
report_metrics()
if error_holder[0] is not None:
raise error_holder[0]
# The outcome is the request's verdict, the channel only its
# tokens: a sequence that ended in FinishReason.ERROR, or whose
# response failed to materialize, still delivered a terminal chunk.
# Every stream the caller did not cancel therefore collects its
# outcome — get() raises the real reason for a failure and returns
# promptly otherwise, since a resident's outcome is published at
# the tick that evicts it, not when the whole batch drains.
handle = state.get("handle")
if (handle is not None and on_metrics is None
and not state.get("cancelled_by_caller")):
handle.get()
return CancellableIterator(_iterate(), _cancel)
[docs]
def validate_context(
self,
messages: List[Dict[str, Any]],
sampling_params: Optional[SamplingParams] = None,
*,
tools: Optional[Sequence[Dict[str, Any]]] = None) -> int:
"""Count a text conversation and reject turns outside the engine limits.
Reserve the requested output budget without truncating the prompt.
Rendered ``tools`` count toward the prompt. This check does not
generate tokens or change the context cache.
"""
params = sampling_params or SamplingParams()
with self._admission(), self._infer_guard():
self._ensure_open()
request = self._make_generation_request(messages,
params,
tools=tools)
count = self._count_prepared_prompt_tokens(request)
if count is None:
raise ValueError(
"Exact context validation requires text input")
if (count > self._max_input_len or count + params.max_tokens
> self._max_kv_cache_capacity):
raise ValueError(
f"Conversation has {count} prompt tokens with "
f"{params.max_tokens} output tokens reserved. Engine limits "
f"are {self._max_input_len} input and "
f"{self._max_kv_cache_capacity} total. Shorten the "
"conversation or rebuild with larger limits: "
"--max-input-len for prompt tokens, "
"--max-kv-cache-capacity for prompt plus output.")
return count
[docs]
def generate_stream_with_audio(
self,
messages: List[Dict[str, Any]],
sampling_params: Optional[SamplingParams] = None,
*,
audio_params: Optional[AudioParams] = None,
prebuilt_request: Optional[Any] = None,
) -> Iterator[StreamDelta]:
"""Stream text and audio deltas for a single Omni request.
Runs the Thinker-Talker streaming pipeline in a background thread.
Text deltas arrive through a ``StreamChannel`` and PCM chunks through
an ``AudioStreamChannel``; the two are interleaved into one generator.
Admission follows generate_stream: the HTTP layer owns the gate when
it passes ``prebuilt_request``; otherwise it is acquired here.
"""
if not self._layout.has_speech:
raise ValueError("Omni audio output not available: talker / "
"code_predictor / code2wav engines not loaded.")
if self._engine is not None:
# In-flight batching: the engine serves text only.
raise ValueError(_IFB_TEXT_ONLY_MESSAGE)
params = sampling_params or SamplingParams()
state = {}
def _cancel():
stream = state.get("stream")
if stream is not None:
stream.close()
for name in ("text_channel", "audio_channel"):
channel = state.get(name)
if channel is not None:
channel.cancel()
def _iterate():
self._ensure_open()
channel = self._rt.StreamChannel.create()
audio_channel = self._rt.AudioStreamChannel()
state["text_channel"] = channel
state["audio_channel"] = audio_channel
channel.set_skip_special_tokens(True)
omni_params = _native_audio_params(self._rt, audio_params
or AudioParams())
sem = None if prebuilt_request is not None else self._admission()
if sem is not None:
sem.acquire()
try:
self._ensure_open()
if prebuilt_request is not None:
request = prebuilt_request
request.stream_channels = [channel]
else:
request = self._make_generation_request(
messages,
params,
stream_channel=channel,
)
except BaseException:
if sem is not None:
sem.release()
raise
def _run():
# Audio and text share one runtime and CUDA stream.
with self._infer_guard():
self._ensure_open()
self._runtime.handle_request_streaming_audio(
request, audio_channel, omni_params)
stream = _pump_channels(self._rt,
_run,
channel,
audio_channel,
sem=sem)
state["stream"] = stream
yield from stream
return CancellableIterator(_iterate(), _cancel)
# ------------------------------------------------------------------
# Server API
# ------------------------------------------------------------------
[docs]
def generate_speech_stream(
self,
text: str,
audio_params: Optional[AudioParams] = None,
) -> Iterator[StreamDelta]:
"""Standalone TTS on the Omni stack: synthesize ``text`` directly.
No Thinker generation pass — the input text goes straight to the
Talker. Yields audio-only StreamDeltas.
"""
if not self._layout.has_speech:
raise ValueError("TTS not available: Omni audio components "
"(talker/code_predictor/code2wav) not loaded")
if self._engine is not None:
# In-flight batching: the engine serves text only.
raise ValueError(_IFB_TEXT_ONLY_MESSAGE)
sem = self._admission()
return _stream_tts(self._rt,
self._runtime,
text,
audio_params or AudioParams(),
sem,
infer_guard=self._infer_guard(),
ensure_open=self._ensure_open)
[docs]
def list_voices(self) -> List[str]:
"""Speaker names accepted as ``voice``; empty when not Omni-capable
(or when in-flight batching left the speech stack unloaded)."""
if not self.speech_available:
return []
self._ensure_open()
return sorted(self._runtime.get_speaker_names())
[docs]
def serve(self,
host: str = "0.0.0.0",
port: int = 8000,
*,
served_model_name: str = "",
api_key: str = "",
reasoning_parser: str = "auto",
tool_call_parser: str = "auto",
enable_auto_tool_choice: bool = False,
max_queued_requests: int = 16,
queue_timeout: float = 600.0,
allowed_local_media_path: Optional[str] = None,
min_request_priority: int = 0) -> None:
"""Start the HTTP frontend for this runtime."""
from ..api.app import run_http_server
from ..config import ApiConfig
from .engine_client import EngineClient
config = ApiConfig(
host=host,
port=port,
served_model_name=served_model_name,
api_key=api_key,
reasoning_parser=reasoning_parser,
tool_call_parser=tool_call_parser,
enable_auto_tool_choice=enable_auto_tool_choice,
max_queued_requests=max_queued_requests,
queue_timeout=queue_timeout,
allowed_local_media_path=allowed_local_media_path or "",
min_request_priority=min_request_priority,
)
run_http_server(EngineClient(self, config), config)
# ------------------------------------------------------------------
# Properties
# ------------------------------------------------------------------
@property
def model_dir(self) -> str:
"""Path to the resolved model checkpoint."""
return self._model_dir
@property
def model_id(self) -> str:
"""User-facing model identifier supplied at initialization."""
return self._model_id
@property
def bundle_dir(self) -> str:
"""Profile-specific engine bundle selected from the cache."""
return self._bundle_dir
@property
def cache_dir(self) -> str:
"""Root containing downloaded checkpoints and built bundles."""
return self._cache_dir
@property
def max_batch_size(self) -> int:
"""Maximum batch size supported by the loaded engine."""
return self._max_batch_size
@property
def max_kv_cache_capacity(self) -> int:
"""Total token capacity supported by the loaded engine."""
return self._max_kv_cache_capacity
@property
def video_capable(self) -> bool:
"""Whether this model bundle supports video input."""
try:
self._video_model_family()
return True
except ValueError:
return False
@property
def has_draft_model(self) -> bool:
"""Whether speculative decoding is active."""
# Under in-flight batching the engine owns the runtime; the deployment
# it was built from says whether a draft engine came with it.
if self._runtime is None:
return (getattr(self, "_engine", None) is not None
and self._layout.engine_type == EngineType.SPEC_DECODE)
return self._runtime.has_draft_model()
@property
def context_cache_enabled(self) -> bool:
"""Whether this runtime reuses matching text prefixes."""
return self._context_cache_config.enabled
@property
def publishes_prefill_state_only(self) -> bool:
"""Whether context reuse can publish only prompt endpoints here:
speculative decoding on a hybrid base."""
cache_config = getattr(self, "_context_cache_config", None)
return (cache_config is not None and cache_config.enabled
and getattr(self, "_hybrid_base", False)
and self.has_draft_model)
[docs]
def get_scheduling_metrics(self):
"""Engine scheduling counters as a dict, or ``None`` on the blocking path."""
engine = getattr(self, "_engine", None)
if engine is None or not hasattr(engine, "metrics"):
return None
m = engine.metrics()
counters = {
field: getattr(m, field)
for field in ("submitted", "refused", "stalls_incompatible",
"stalls_guided", "stalls_exclusive",
"stalls_no_capacity", "admitted_mid_flight",
"completed", "cancelled", "failed",
"queue_latency_max_us", "decode_steps",
"decode_graph_steps", "prompt_steps", "mixed_steps",
"decode_subset_steps", "step_sequences",
"step_prompt_chunks", "partial_prompt_chunks",
"step_decode_rows", "step_tokens",
"step_sampled_rows", "step_time_total_us",
"step_time_max_us", "paused_resident_steps",
"compact_refusals")
}
counters["queue_latency_avg_us"] = (m.queue_latency_total_us //
m.queue_latency_count
if m.queue_latency_count else 0)
steps = m.decode_steps + m.prompt_steps + m.decode_subset_steps
counters["step_time_avg_us"] = (m.step_time_total_us //
steps if steps else 0)
# Live gauges alongside the counters: current depth is what an
# operator reading /health actually wants first.
counters["queued"] = getattr(engine, "queued", 0)
counters["resident"] = getattr(engine, "resident", 0)
return counters
[docs]
def engine_unavailable_reason(self) -> Optional[str]:
"""Why the native runtime no longer serves requests, or ``None`` while reusable."""
engine = getattr(self, "_engine", None)
if engine is None:
return getattr(getattr(self, "_runtime", None),
"unavailable_reason", None) or None
status = engine.status()
return None if status.serving else status.unavailable_reason
[docs]
def get_engine_status(self):
"""Native failure and in-flight batching state for /health.
``serving`` turns False once the engine fails every request, with ``unavailable_reason``. The scheduling
fields are the policy in effect, which a plugin refusing a compact step under ``auto`` changes to
prefillFirst over single-cohort steps: the mode, how it was chosen (``requested`` or ``auto``) and, under
``auto``, why; ``preemption`` as applied, the mixed-step budgets (0 outside mixed), and ``warnings`` for
configuration accepted but not applied as given.
"""
engine = getattr(self, "_engine", None)
if engine is None:
reason = self.engine_unavailable_reason()
return {
"serving": False,
"unavailable_reason": reason
} if reason else None
status = engine.status()
policy = engine.resolved_policy
compact_reason = status.compact_unavailable_reason
return {
"serving": status.serving,
"unavailable_reason": status.unavailable_reason or None,
"scheduling_mode": policy.mode,
"scheduling_mode_source": policy.mode_source,
"scheduling_reason": policy.reason or None,
"execution": policy.execution,
"scheduling_policy": policy.scheduling_policy,
"preemption": policy.preemption,
"max_num_tokens": policy.max_num_tokens,
"max_chunk_tokens": policy.max_chunk_tokens,
"priority_reserved_slots": policy.priority_reserved_slots,
"max_pause_steps": policy.max_pause_steps,
"warnings": policy.warnings,
"compact_unavailable_reason": compact_reason or None,
}
[docs]
def get_context_cache_metrics(self):
"""Return native reuse counters, or ``None`` when reuse is disabled."""
engine = getattr(self, "_engine", None)
if engine is not None:
# The actor's copy as of its last admission or step; reading it
# takes no lock the engine's requests contend on.
return engine.context_cache_metrics()
if self._runtime is None:
return None
with self._infer_guard():
self._ensure_open()
return self._runtime.get_context_cache_metrics()
[docs]
def get_context_cache_counters(self):
"""Context-cache counters as a dict for ``/health``, or ``None``.
``None`` also on the blocking path: reading the counters there waits
for the running request, which a health probe must not do.
"""
engine = getattr(self, "_engine", None)
if engine is None:
return None
cc = engine.context_cache_metrics()
if cc is None:
return None
counters = {
field: getattr(cc, field)
for field in _CONTEXT_CACHE_COUNTER_FIELDS
}
for pool in _CONTEXT_CACHE_POOL_FIELDS:
gauge = getattr(cc, pool)
counters[pool] = {"free": gauge.free, "capacity": gauge.capacity}
return counters
@property
def bundle_layout(self) -> BundleLayout:
"""Immutable component contract for the selected model bundle."""
return self._layout
[docs]
class TTS:
"""Checkpoint-direct serving for a model-owned TTS component stack."""
runtime_kind = "tts"
has_draft_model = False
[docs]
def __init__(
self,
model: str,
*,
cache_dir: str = "",
engine_cache_max_size_gb: float = 50.0,
clear_engine_cache: bool = False,
max_input_len: int = _DEFAULT_MAX_INPUT_LEN,
max_batch_size: int = _DEFAULT_MAX_BATCH_SIZE,
max_kv_cache_capacity: int = _DEFAULT_MAX_KV_CACHE_CAPACITY,
build_options: Optional["BuildOptions"] = None,
) -> None:
if not model:
raise ValueError("'model' must be provided")
if (isinstance(engine_cache_max_size_gb, bool)
or not math.isfinite(engine_cache_max_size_gb)
or engine_cache_max_size_gb <= 0):
raise ValueError("engine_cache_max_size_gb must be positive")
from .engine_build import BuildOptions, cache_root, prepare_model
options = build_options or BuildOptions(
max_input_len=max_input_len,
max_batch_size=max_batch_size,
max_kv_cache_capacity=max_kv_cache_capacity,
)
if options.spec_type != "none":
raise ValueError(
"standalone TTS models do not support speculative decoding")
prepared = prepare_model(
model,
cache_dir,
options,
max_cache_size_bytes=int(engine_cache_max_size_gb * (1 << 30)),
clear_cache=clear_engine_cache,
)
self._layout = inspect_bundle(prepared.bundle_dir)
if not self._layout.has_speech:
raise ValueError(
f"model {model!r} does not provide a complete TTS runtime")
self._model_dir = prepared.model_dir
self._model_id = _derive_model_id(model)
self._cache_dir = cache_root(cache_dir)
self._bundle_dir = prepared.bundle_dir
self._rt = _import_runtime()
logger.info("Loading TTS runtime from %s", prepared.bundle_dir)
self._runtime = self._rt.TTSRuntime(
talker_engine_dir=self._layout.talker_dir,
code_predictor_engine_dir=self._layout.code_predictor_dir,
code2wav_engine_dir=self._layout.code2wav_dir,
tokenizer_dir=self._layout.talker_dir,
checkpoint_dir=prepared.model_dir,
)
logger.info("TTS runtime ready")
self._admission_sem = threading.Semaphore(1)
self._close_lock = threading.Lock()
self._closed = False
def _admission(self):
"""Per-instance admission gate (mirrors LLM._admission)."""
return self._admission_sem
[docs]
def generate_speech_stream(
self,
text: str,
audio_params: Optional[AudioParams] = None,
) -> Iterator[StreamDelta]:
"""Synthesize ``text``; yields audio-only StreamDeltas."""
sem = self._admission()
return _stream_tts(self._rt,
self._runtime,
text,
audio_params or AudioParams(),
sem,
ensure_open=self._ensure_open)
def _ensure_open(self) -> None:
if self._closed or self._runtime is None:
raise RuntimeError("Edge-LLM runtime is closed")
[docs]
def list_voices(self) -> List[str]:
"""Speaker names accepted as ``voice``."""
self._ensure_open()
return sorted(self._runtime.get_speaker_names())
[docs]
def close(self) -> None:
"""Drain active speech generation and release native resources."""
with self._close_lock:
if self._closed:
return
self._closed = True
with self._admission_sem:
self._runtime = None
def __enter__(self) -> "TTS":
self._ensure_open()
return self
def __exit__(self, _exc_type, _exc, _traceback) -> None:
self.close()
@property
def model_id(self) -> str:
return self._model_id
@property
def model_dir(self) -> str:
return self._model_dir
@property
def bundle_dir(self) -> str:
return self._bundle_dir
@property
def cache_dir(self) -> str:
return self._cache_dir
@property
def bundle_layout(self) -> BundleLayout:
"""Immutable component contract for the selected model bundle."""
return self._layout
[docs]
def serve(self, host: str = "0.0.0.0", port: int = 8000) -> None:
"""Start the HTTP server (speech endpoint only)."""
from ..api.app import run_http_server
from ..config import ApiConfig
from .engine_client import EngineClient
config = ApiConfig(host=host, port=port)
run_http_server(EngineClient(self, config), config)
[docs]
def load_model(**kwargs):
"""Select the model-specific runtime from provider checkpoint metadata."""
from .engine_build import resolve_model_dir
original_model = kwargs["model"]
resolved = resolve_model_dir(original_model, kwargs.get("cache_dir", ""))
with open(os.path.join(resolved, "config.json"), encoding="utf-8") as file:
model_type = json.load(file).get("model_type")
if model_type == "qwen3_tts":
runtime_class = TTS
for name in ("draft_top_k", "draft_step", "proposal_tokens",
"verify_tokens"):
if kwargs.pop(name, None) is not None:
raise ValueError(
f"standalone TTS models do not support {name}")
if kwargs.pop("speculative_config", None):
raise ValueError(
"standalone TTS models do not support speculative decoding")
if kwargs.pop("enable_in_flight_batching", False):
raise _ifb_unsupported_error(
ifb_unsupported_reason(None, model_type=model_type))
scheduling = {
name: kwargs.pop(name, default)
for name, default in IFB_SCHEDULING_DEFAULTS.items()
}
validate_ifb_scheduling_options(False, scheduling)
ple_config = PleRuntimeConfig.parse(kwargs.pop("ple_config", None))
if ple_config != PleRuntimeConfig():
raise ValueError("standalone TTS models do not use PLE")
context_cache = ContextCacheConfig.parse(
kwargs.pop("context_cache_config", None))
if context_cache.enabled:
raise ValueError(
"standalone TTS models do not use a KV context cache")
else:
runtime_class = LLM
runtime = runtime_class(**{**kwargs, "model": resolved})
runtime._model_id = _derive_model_id(original_model)
return runtime
# ---------------------------------------------------------------------------
# Message conversion & image loading
# ---------------------------------------------------------------------------
def finish_reason_name(rt_module, reason) -> Optional[str]:
"""Map a C++ FinishReason enum value to its OpenAI-compatible string.
NOT_FINISHED maps to None — reaching this function with a non-terminal
reason indicates a bug; surfacing None instead of silently returning "stop"
makes it visible. The fallback "stop" catches truly-unknown enum values
(e.g. future C++ enum additions). STOP_WORDS and END_ID both map to "stop"
since OpenAI does not distinguish them.
"""
return {
rt_module.FinishReason.NOT_FINISHED: None,
rt_module.FinishReason.END_ID: "stop",
rt_module.FinishReason.LENGTH: "length",
rt_module.FinishReason.CANCELLED: "cancelled",
rt_module.FinishReason.ERROR: "error",
rt_module.FinishReason.STOP_WORDS: "stop",
}.get(reason, "stop")
def _tool_call_arguments(arguments: Any) -> Dict[str, Any]:
"""Return assistant tool-call arguments as the object provider templates
read. OpenAI clients send a JSON string; as in vLLM, an empty or
non-object value becomes an empty object."""
if isinstance(arguments, str) and arguments:
try:
arguments = json.loads(arguments)
except json.JSONDecodeError as error:
raise ValueError(f"Invalid JSON in assistant tool-call arguments: "
f"{error.msg}") from error
if isinstance(arguments, dict):
return arguments
if arguments:
logger.warning(
"Rendering non-object assistant tool-call arguments of type %s "
"as an empty object",
type(arguments).__name__)
return {}
def _convert_messages_to_cpp(rt_module, messages: List[Dict[str, Any]]):
"""Convert Python message dicts to C++ Message objects."""
cpp_messages = []
for msg in messages:
cpp_msg = rt_module.Message()
cpp_msg.role = msg["role"]
reasoning_key = ("reasoning_content" if "reasoning_content" in msg else
"reasoning" if "reasoning" in msg else None)
cpp_msg.has_reasoning_content = reasoning_key is not None
cpp_msg.reasoning_content = (str(msg.get(reasoning_key) or "")
if reasoning_key else "")
cpp_msg.tool_call_id = str(msg.get("tool_call_id") or "")
cpp_msg.name = str(msg.get("name") or "")
cpp_msg.has_content = "content" in msg
cpp_msg.content_is_array = isinstance(msg.get("content"), list)
cpp_msg.content_is_null = cpp_msg.has_content and msg["content"] is None
raw_tool_calls = msg.get("tool_calls") or []
if msg.get("function_call") and not raw_tool_calls:
raw_tool_calls = [{
"type": "function",
"function": msg["function_call"],
}]
# vLLM drops an empty tool_calls field so provider templates keep the
# ordinary assistant-message path.
cpp_msg.has_tool_calls = bool(raw_tool_calls)
cpp_tool_calls = []
for raw_call in raw_tool_calls:
function = raw_call.get("function", raw_call)
call = rt_module.MessageToolCall()
call.id = str(raw_call.get("id") or "")
call.type = str(raw_call.get("type") or "function")
call.name = str(function.get("name") or "")
arguments = _tool_call_arguments(function.get("arguments"))
call.arguments = json.dumps(arguments,
ensure_ascii=False,
separators=(",", ":"))
cpp_tool_calls.append(call)
cpp_msg.tool_calls = cpp_tool_calls
content = msg.get("content")
contents_list = []
if isinstance(content, str):
contents_list.append(rt_module.MessageContent("text", content))
elif content is not None and not isinstance(content, list):
raise ValueError(
"Message content must be a string, an array, or null")
elif isinstance(content, list):
for item in content:
if isinstance(item, str):
contents_list.append(rt_module.MessageContent(
"text", item))
elif isinstance(item, dict):
ct = item.get("type", "text")
if ct in ("text", "input_text"):
contents_list.append(
rt_module.MessageContent(
"text",
item.get("text", ""),
))
elif ct in ("image", "image_url"):
contents_list.append(
rt_module.MessageContent("image", ""))
elif ct in ("video", "video_url"):
# Frames are decoded out-of-band by _load_image_buffers; the chat
# template expands this placeholder into the video triplet and the
# ViT runner keys off ImageData.isVideo.
contents_list.append(
rt_module.MessageContent("video", ""))
elif ct in ("audio", "input_audio", "audio_url"):
# Audio bytes are decoded out-of-band by
# `_load_audio_buffers`; the chat template just emits
# an opaque audio placeholder here. The per-model
# audio runner expands that into model-specific
# special tokens (Qwen3: <|audio_start|> +
# N×<|audio_pad|> + <|audio_end|>; Nemotron-Omni:
# N×<so_embedding>).
contents_list.append(
rt_module.MessageContent("audio", ""))
else:
raise ValueError(f"Unsupported content type: {ct}")
else:
raise ValueError(
"Message content array items must be strings or objects"
)
cpp_msg.contents = contents_list
cpp_messages.append(cpp_msg)
return cpp_messages
def _convert_tools_to_cpp(rt_module, tool_config: ToolConfig):
"""Convert validated OpenAI function tools to the native render contract."""
if tool_config.tool_choice == "none":
return []
result = []
for raw_tool in tool_config.tools:
function = raw_tool["function"]
tool = rt_module.ToolDefinition()
tool.name = function["name"]
tool.description = function.get("description", "")
tool.has_description = "description" in function
tool.parameters = json.dumps(function.get("parameters", {}),
ensure_ascii=False,
separators=(",", ":"))
tool.has_parameters = "parameters" in function
tool.strict = bool(function.get("strict", False))
tool.has_strict = "strict" in function
result.append(tool)
return result
def _convert_tool_choice_to_cpp(rt_module, tool_config: ToolConfig):
choice = rt_module.ToolChoice()
modes = {
"none": rt_module.ToolChoiceMode.NONE,
"auto": rt_module.ToolChoiceMode.AUTO,
"required": rt_module.ToolChoiceMode.REQUIRED,
"function": rt_module.ToolChoiceMode.FUNCTION,
}
choice.mode = modes[tool_config.tool_choice]
choice.function_name = tool_config.forced_name or ""
return choice
def _load_image_buffers(rt_module,
messages: List[Dict[str, Any]],
video_family_fn=lambda: "qwen",
video_frame_limits_fn=lambda: {}):
"""Build the ordered ImageData list for the messages: images and videos
share one list the C++ runner matches positionally against the
<|image_pad|> / <|video_pad|> placeholders, so append in message order."""
from ..media.video_sampling import VisualProfileCapacityError
images = []
items = [
item for msg in messages if isinstance(msg.get("content"), list)
for item in msg["content"] if isinstance(item, dict)
]
from ..media.media_source import resolve_image_message
image_sources = {
id(item): resolve_image_message(item)
for item in items if item.get("type") in ("image", "image_url")
}
# Videos and images share one engine token profile: track the remaining
# budget so multiple media cannot each claim full capacity. Reading limits
# is lazy so text-only requests never touch the visual bundle.
has_video = any(
item.get("type") in ("video", "video_url") for item in items)
limits = video_frame_limits_fn() if has_video or image_sources else {}
if has_video:
family = video_family_fn()
else:
model_type = limits.get("model_type", "")
family = _visual_model_family(model_type)
if family not in ("internvl", "muse", "qwen"):
# Image-only runners with different resize rules retain their C++
# validation until a matching server-side estimator is available.
family = "qwen"
limits = {}
if has_video and family == "nemotron":
# The C++ Nemotron video path handles exactly one video and no mixed-in
# images per request (batch of one); reject other layouts here rather
# than letting them fail inside the runner.
n_videos = sum(1 for it in items
if it.get("type") in ("video", "video_url"))
n_images = sum(1 for it in items
if it.get("type") in ("image", "image_url"))
if n_videos > 1 or n_images > 0:
raise ValueError(
"Nemotron-Omni video requests support exactly one video and no "
f"images (got {n_videos} videos, {n_images} images)")
budget = limits.get("max_image_tokens") if limits else None
video_tokens = 0
# Pre-pruning token count for the engine-minimum check: the ViT processes
# every tubelet, so Nemotron's EVS-pruned estimate would understate what the
# min-profile actually receives. Non-EVS families track the same value.
video_raw_tokens = 0
# Request-wide decoded-pixel budget: several videos each under the
# per-video ceiling must not jointly exhaust host memory.
pixel_budget = None
# Request-wide cu_seqlens group budget (Qwen only; InternVL has no
# cu_seqlens binding): use builder-recorded capacity or derive it from the
# current component profile.
cu_budget = None
if limits and family != "internvl":
cu_budget = (limits.get("max_cu_seqlen_groups")
or limits["max_image_tokens"] //
max(1, limits.get("min_image_tokens", 1)))
image_upper = 0
# Phase 1: reserve every image up front so the video sampler's budget is
# order-independent ([image, video] and [video, image] behave identically).
if budget is not None:
from ..media.video_sampling import estimate_image_tokens
for item in items:
if item.get("type") not in ("image", "image_url"):
continue
source = image_sources[id(item)]
if isinstance(source, bytes) or (isinstance(source, str)
and os.path.isfile(source)):
est = estimate_image_tokens(source,
family,
limits,
do_resize=bool(
item.get("do_resize", True)))
else:
est = int(limits.get("max_image_tokens_per_image", 0))
image_upper += est
budget -= est
if cu_budget is not None:
# One cu_seqlens entry per image (Qwen families only;
# InternVL has no cu_seqlens binding).
cu_budget -= 1
# Phase 2: build the buffers in original message order (the C++ runner
# matches them positionally against the placeholders).
for item in items:
itype = item.get("type")
if itype in ("image", "image_url"):
source = image_sources[id(item)]
image = (rt_module.load_image_from_bytes(source) if isinstance(
source, bytes) else rt_module.load_image_from_path(source))
image.do_resize = bool(item.get("do_resize", True))
images.append(image)
elif itype in ("video", "video_url"):
from ..media.video_sampling import (MAX_DECODE_PIXELS,
load_video_buffer)
if pixel_budget is None:
pixel_budget = MAX_DECODE_PIXELS
buffer, est_tokens, used_px, used_groups = load_video_buffer(
rt_module,
item,
family,
frame_limits=limits,
budget=budget,
pixel_budget=pixel_budget,
cu_budget=cu_budget)
images.append(buffer)
video_tokens += est_tokens
raw_tokens = est_tokens
if family == "nemotron":
from ..media.video_sampling import _nemotron_tubelet_geometry
geom = _nemotron_tubelet_geometry(limits)
if geom:
t_frames, tokens_per_tubelet, _q = geom
raw_tokens = (-(-buffer.frames // t_frames)) \
* tokens_per_tubelet
video_raw_tokens += raw_tokens
pixel_budget -= used_px
if cu_budget is not None:
cu_budget -= used_groups
if budget is not None:
budget -= est_tokens
# Engine bounds are request-wide (all media accumulate in one ViT batch),
# so validate after the loop: two videos jointly reaching the minimum are
# fine, one alone may not be.
# Only image-only overflows map to HTTP 413; video paths still raise 400.
capacity_error = (VisualProfileCapacityError
if image_sources and not has_video else ValueError)
if cu_budget is not None and cu_budget < 0:
raise capacity_error(
"request media exceed the visual engine's cu_seqlens capacity; "
"reduce the media count")
if budget is not None and budget < 0:
estimated_tokens = image_upper + video_tokens
raise capacity_error(
f"request media need an estimated {estimated_tokens} visual "
"tokens, exceeding the visual engine profile "
f"max_image_tokens={limits['max_image_tokens']}; reduce the media "
"or rebuild the visual engine with a larger --maxImageTokens")
if (video_raw_tokens or image_upper) and limits and \
limits.get("min_image_tokens"):
# The engine minimum is request-wide; the upper estimate is pre-EVS
# (raw tubelets for Nemotron). It can fall short for a too-short clip or
# do_resize=false media; resized per-item images are floored above this.
upper_tokens = video_raw_tokens + image_upper
if upper_tokens < limits["min_image_tokens"]:
raise ValueError(
f"request media yield ~{upper_tokens} visual tokens but the "
f"engine needs at least {limits['min_image_tokens']}; use "
"longer videos or raise nframes/fps")
return images
def _load_audio_buffers(rt_module, messages: List[Dict[str, Any]]):
"""Load audio content from messages into AudioData buffers.
Returns an empty list when no audio is present, keeping the byte-identical
fast path for text-only and image-only requests.
"""
from ..media.audio_preprocess import load_audio_buffers
return load_audio_buffers(rt_module, messages)