# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Validated launch configuration for the inference server."""
import argparse
import json
import math
import re
from dataclasses import dataclass, field
from typing import Any, Dict, Mapping, Optional, Sequence, Union
_INT32_MAX = 2**31 - 1
class ServerConfigError(ValueError):
"""Raised when a launch option cannot be honored by Edge-LLM."""
# In-flight batching scheduling options at their defaults; any other value needs
# --enable-in-flight-batching.
IFB_SCHEDULING_DEFAULTS = {
"ifb_scheduling_mode": "auto",
"max_num_tokens": 0,
"max_chunk_tokens": 0,
"scheduling_policy": "fcfs",
"preemption": "none",
"priority_reserved_slots": 0,
"max_pause_steps": 0,
}
# Option names in launch errors, keyed by the spellings the Python API and the
# C++ RequestEngine use; scheduling_option_flags() turns them into CLI flags.
_SCHEDULING_FLAG_NAMES = {
spelled: "--" + name.replace("_", "-")
for name in ("ifb_scheduling_mode", "max_num_tokens", "max_chunk_tokens",
"scheduling_policy", "priority_reserved_slots",
"max_pause_steps", "max_batch_size")
for spelled in (name, re.sub(r"_(\w)", lambda m: m.group(1).upper(), name))
}
# "preemption" is also an English word, so only "preemption=..." names the
# option.
_SCHEDULING_NAME_RE = re.compile(
r"(?<![\w-])(?:(" +
"|".join(sorted(_SCHEDULING_FLAG_NAMES, key=len, reverse=True)) +
r")\b|(preemption)(?==))(=)?")
def scheduling_option_flags(message: str) -> str:
"""Rewrite scheduling option names in a launch error as the server's
command-line flags (``ifbSchedulingMode=mixed`` -> ``--ifb-scheduling-mode
mixed``)."""
def flag(match: "re.Match[str]") -> str:
name = match.group(1) or match.group(2)
spelled = _SCHEDULING_FLAG_NAMES.get(name, "--preemption")
return spelled + (" " if match.group(3) else "")
return _SCHEDULING_NAME_RE.sub(flag, message)
def validate_ifb_scheduling_options(
enable_in_flight_batching: bool,
options: Mapping[str, Any],
max_batch_size: Optional[int] = None) -> None:
"""Reject scheduling options before the engine is built.
Without in-flight batching every option must keep its default. With it,
the options must be int32 integers and pass the RequestEngine's own
validation for an engine built for ``max_batch_size`` (None when that is
not known before the build, which leaves the check to the engine's
startup). The step budgets and what the deployment supports are checked
when the engine starts.
"""
if not enable_in_flight_batching:
changed = sorted(name
for name, default in IFB_SCHEDULING_DEFAULTS.items()
if options[name] != default)
if changed:
raise ServerConfigError(
", ".join(changed) +
" only apply with --enable-in-flight-batching")
return
for name in ("max_num_tokens", "max_chunk_tokens",
"priority_reserved_slots", "max_pause_steps"):
value = options[name]
if (isinstance(value, bool) or not isinstance(value, int)
or not -(2**31) <= value <= _INT32_MAX):
raise ServerConfigError(f"{name} must be an int32 integer")
if max_batch_size is None:
return
from .runtime.engine import _import_runtime
try:
_import_runtime().validate_scheduling_options(
**options, max_batch_size=max_batch_size)
except ValueError as exc:
raise ServerConfigError(str(exc)) from exc
@dataclass(frozen=True)
class SpeculativeConfig:
"""Speculative model pair, runtime proposal length, and tree shape."""
method: str
num_speculative_tokens: Optional[int] = None
draft_model: str = ""
draft_top_k: Optional[int] = None
verify_tokens: Optional[int] = None
@classmethod
def parse(
cls, value: Union[str, Mapping[str, Any], "SpeculativeConfig", None]
) -> Optional["SpeculativeConfig"]:
if not value:
return None
if isinstance(value, cls):
raw = {
"method": value.method,
"num_speculative_tokens": value.num_speculative_tokens,
# An absent draft model is stored as "", which re-parsing would
# reject as a malformed model id. MTP is the only method that
# legitimately has none, so re-parsing must round-trip.
"model": value.draft_model or None,
"draft_top_k": value.draft_top_k,
"verify_tokens": value.verify_tokens,
}
elif isinstance(value, str):
try:
raw = json.loads(value)
except json.JSONDecodeError as exc:
raise ServerConfigError(
"--speculative-config must be a JSON object") from exc
else:
raw = dict(value)
if not isinstance(raw, dict):
raise ServerConfigError(
"--speculative-config must be a JSON object")
supported_keys = {
"method", "num_speculative_tokens", "model", "draft_top_k",
"verify_tokens"
}
unsupported = sorted(set(raw) - supported_keys)
if unsupported:
raise ServerConfigError("unsupported --speculative-config keys: " +
", ".join(unsupported))
method = raw.get("method")
if not isinstance(method, str) or method not in {
"eagle3", "mtp", "dflash", "jetspec", "domino", "dspark"
}:
raise ServerConfigError(
"--speculative-config.method must be one of eagle3, mtp, "
"dflash, jetspec, domino, or dspark")
num_tokens = _optional_int_at_least(raw, "num_speculative_tokens", 1)
draft_top_k = _optional_int_at_least(raw, "draft_top_k", 1)
verify_tokens = _optional_int_at_least(raw, "verify_tokens", 2)
draft_model = raw.get("model")
if draft_model is not None and (not isinstance(draft_model, str)
or not draft_model.strip()):
raise ServerConfigError(
"--speculative-config.model must be a checkpoint path or "
"Hugging Face model ID")
if method in {"eagle3", "dflash", "jetspec", "domino", "dspark"
} and not draft_model:
raise ServerConfigError(
f"--speculative-config.method {method!r} requires "
"speculative-config.model")
return cls(method=method,
num_speculative_tokens=num_tokens,
draft_model=draft_model or "",
draft_top_k=draft_top_k,
verify_tokens=verify_tokens)
def _optional_int_at_least(raw: Mapping[str, Any], key: str,
minimum: int) -> Optional[int]:
value = raw.get(key)
if value is not None and (isinstance(value, bool) or
not isinstance(value, int) or value < minimum):
raise ServerConfigError(
f"--speculative-config.{key} must be an integer >= {minimum}")
return value
#: Requests that may wait behind the running batch before the server refuses
#: (HTTP 429) and the direct Python API blocks. One number for both entry points.
DEFAULT_MAX_QUEUED_REQUESTS = 16
[docs]
@dataclass(frozen=True)
class ContextCacheConfig:
"""Deployment-scoped KV-cache reuse configuration."""
enabled: bool = False
max_records: int = 1024
# Total snapshots on a model with recurrent layers; 0 keeps 4 x max batch size.
max_snapshots: int = 0
def __post_init__(self) -> None:
if not isinstance(self.enabled, bool):
raise ServerConfigError("context_cache.enabled must be boolean")
for name in ("max_records", "max_snapshots"):
value = getattr(self, name)
if isinstance(value,
bool) or not isinstance(value, int) or value < 0:
raise ServerConfigError(
f"context_cache.{name} must be a non-negative integer")
if value > _INT32_MAX:
raise ServerConfigError(
f"context_cache.{name} must fit a signed 32-bit integer")
if not self.enabled and (self.max_records != 1024
or self.max_snapshots):
raise ServerConfigError(
"context-cache tuning requires context reuse to be enabled")
[docs]
@classmethod
def parse(
cls,
value: Optional[Union[Mapping[str, Any], "ContextCacheConfig"]],
) -> "ContextCacheConfig":
"""Normalize an HLAPI mapping without accepting ignored keys."""
if value is None:
return cls()
if isinstance(value, cls):
return value
if not isinstance(value, Mapping):
raise ServerConfigError(
"context_cache_config must be a mapping or ContextCacheConfig")
raw = dict(value)
supported = {"enabled", "max_records", "max_snapshots"}
unsupported = sorted(set(raw) - supported)
if unsupported:
raise ServerConfigError("unsupported context-cache keys: " +
", ".join(unsupported))
return cls(**raw)
[docs]
@dataclass(frozen=True)
class PleRuntimeConfig:
"""Runtime PLE residency and source selection; independent of engine builds."""
residency: str = "resident"
cache_slots: int = 0
file_path: str = ""
def __post_init__(self) -> None:
if self.residency not in ("resident", "cached"):
raise ServerConfigError("PLE residency must be resident or cached")
if (isinstance(self.cache_slots, bool)
or not isinstance(self.cache_slots, int)
or not 0 <= self.cache_slots <= _INT32_MAX):
raise ServerConfigError(
"PLE cache_slots must be a non-negative signed 32-bit integer")
if self.residency == "cached" and self.cache_slots < 2:
raise ServerConfigError(
"Cached PLE requires cache_slots >= 2, including the zero slot"
)
if not isinstance(self.file_path, str):
raise ServerConfigError("PLE file_path must be a string")
[docs]
@classmethod
def parse(
cls, value: Optional[Union[Mapping[str, Any], "PleRuntimeConfig"]]
) -> "PleRuntimeConfig":
"""Normalize Python settings without accepting ignored keys."""
if value is None:
return cls()
if isinstance(value, cls):
return value
if not isinstance(value, Mapping):
raise ServerConfigError(
"ple_config must be a mapping or PleRuntimeConfig")
raw = dict(value)
unsupported = sorted(
set(raw) - {"residency", "cache_slots", "file_path"})
if unsupported:
raise ServerConfigError("unsupported PLE keys: " +
", ".join(unsupported))
return cls(**raw)
@dataclass(frozen=True)
class ModelConfig:
"""Checkpoint, build-cache, and runtime-profile options."""
model: str
cache_dir: str = ""
engine_cache_max_size_gb: float = 50.0
clear_engine_cache: bool = False
max_input_len: int = 4096
max_batch_size: int = 1
max_kv_cache_capacity: int = 8192
max_spec_verify_tokens: Optional[int] = None
max_spec_draft_tokens: Optional[int] = None
max_image_tokens: Optional[int] = None
max_image_tokens_per_image: Optional[int] = None
draft_top_k: Optional[int] = None
draft_step: Optional[int] = None
proposal_tokens: Optional[int] = None
verify_tokens: Optional[int] = None
speculative_config: Optional[SpeculativeConfig] = None
context_cache_config: ContextCacheConfig = field(
default_factory=ContextCacheConfig)
ple_config: PleRuntimeConfig = field(default_factory=PleRuntimeConfig)
enable_in_flight_batching: bool = False
ifb_scheduling_mode: str = "auto"
max_num_tokens: int = 0
max_chunk_tokens: int = 0
scheduling_policy: str = "fcfs"
preemption: str = "none"
priority_reserved_slots: int = 0
max_pause_steps: int = 0
def __post_init__(self) -> None:
if (isinstance(self.engine_cache_max_size_gb, bool)
or not math.isfinite(self.engine_cache_max_size_gb)
or self.engine_cache_max_size_gb <= 0):
raise ServerConfigError(
"engine_cache_max_size_gb must be positive")
validate_ifb_scheduling_options(self.enable_in_flight_batching,
self.ifb_scheduling_kwargs(),
self.max_batch_size)
def ifb_scheduling_kwargs(self) -> Dict[str, Any]:
return {name: getattr(self, name) for name in IFB_SCHEDULING_DEFAULTS}
def llm_kwargs(self) -> Dict[str, Any]:
return {
"model": self.model,
"cache_dir": self.cache_dir,
"engine_cache_max_size_gb": self.engine_cache_max_size_gb,
"clear_engine_cache": self.clear_engine_cache,
"max_input_len": self.max_input_len,
"max_batch_size": self.max_batch_size,
"max_kv_cache_capacity": self.max_kv_cache_capacity,
"max_spec_verify_tokens": self.max_spec_verify_tokens,
"max_spec_draft_tokens": self.max_spec_draft_tokens,
"max_image_tokens": self.max_image_tokens,
"max_image_tokens_per_image": self.max_image_tokens_per_image,
"draft_top_k": self.draft_top_k,
"draft_step": self.draft_step,
"proposal_tokens": self.proposal_tokens,
"verify_tokens": self.verify_tokens,
"speculative_config": self.speculative_config,
"context_cache_config": self.context_cache_config,
"ple_config": self.ple_config,
"enable_in_flight_batching": self.enable_in_flight_batching,
**self.ifb_scheduling_kwargs(),
}
@dataclass(frozen=True)
class ApiConfig:
"""HTTP protocol and request-lifecycle options."""
host: str = "0.0.0.0"
port: int = 8000
served_model_name: str = ""
api_key: str = ""
reasoning_parser: str = "auto"
tool_call_parser: str = "auto"
enable_auto_tool_choice: bool = False
max_queued_requests: int = DEFAULT_MAX_QUEUED_REQUESTS
queue_timeout: float = 600.0
allowed_local_media_path: str = ""
log_level: str = "info"
#: A request whose priority is below this is refused with HTTP 400. At
#: most 0, so a request without a priority (0) is always accepted.
min_request_priority: int = 0
def __post_init__(self) -> None:
if self.max_queued_requests < 0:
raise ServerConfigError("max_queued_requests must be non-negative")
if self.queue_timeout <= 0:
raise ServerConfigError("queue_timeout must be positive")
if (isinstance(self.min_request_priority, bool)
or not isinstance(self.min_request_priority, int)
or not -(2**31) <= self.min_request_priority <= 0):
raise ServerConfigError(
"min_request_priority must be an int32 integer <= 0: a "
"positive minimum would refuse every request without a "
"priority, including every /v1/responses and /v1/messages "
"request")
@dataclass(frozen=True)
class ServerConfig:
"""Complete validated launch configuration."""
model: ModelConfig
api: ApiConfig
def _positive_int(value: str) -> int:
parsed = int(value)
if parsed < 1:
raise argparse.ArgumentTypeError("must be a positive integer")
return parsed
def _non_negative_int(value: str) -> int:
parsed = int(value)
if parsed < 0:
raise argparse.ArgumentTypeError("must be a non-negative integer")
return parsed
def _positive_float(value: str) -> float:
parsed = float(value)
if not math.isfinite(parsed) or parsed <= 0:
raise argparse.ArgumentTypeError("must be positive")
return parsed
def _bounded_port(value: str) -> int:
parsed = int(value)
if not 1 <= parsed <= 65535:
raise argparse.ArgumentTypeError("must be in [1, 65535]")
return parsed
def create_argument_parser(
prog="tensorrt-edgellm-serve") -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
prog=prog,
description="TensorRT Edge-LLM OpenAI-compatible server",
)
parser.add_argument(
"model",
help="Hugging Face model ID or local checkpoint directory",
)
api = parser.add_argument_group("HTTP server")
api.add_argument("--host",
default="0.0.0.0",
help="Address to bind (default: %(default)s)")
api.add_argument("--port",
type=_bounded_port,
default=8000,
help="TCP port to listen on (default: %(default)s)")
api.add_argument(
"--served-model-name",
default="",
help=("API model name. Defaults to the Hugging Face model ID or local "
"checkpoint directory name"))
api.add_argument(
"--api-key",
default="",
help=("Key required for /v1/ routes via Bearer authorization or "
"x-api-key. Unset disables authentication"))
api.add_argument("--reasoning-parser",
choices=("auto", "none", "qwen3", "deepseek_r1",
"muse_glimmer", "nemotron"),
default="auto",
help="Parser for reasoning output (default: %(default)s)")
api.add_argument(
"--tool-call-parser",
choices=("auto", "generic", "hermes", "qwen3_xml", "nemotron",
"openai"),
default="auto",
help="Parser for model-generated tool calls (default: %(default)s)")
api.add_argument("--enable-auto-tool-choice",
action="store_true",
help="Allow the model to select tools automatically")
api.add_argument(
"--min-request-priority",
type=int,
default=0,
help="Lowest request priority the server accepts, at most 0; a "
"request with a lower priority gets HTTP 400. The default 0 refuses "
"the urgent (negative) priorities, and with them the slots "
"--priority-reserved-slots keeps; set it below 0 (for example -1) "
"to let clients use them. /v1/responses and /v1/messages requests "
"carry no priority and run at 0.",
)
api.add_argument(
"--max-queued-requests",
type=int,
default=16,
help="Maximum number of waiting requests (default: %(default)s)")
api.add_argument(
"--queue-timeout",
type=float,
default=600.0,
help="Maximum queue wait in seconds (default: %(default)s)")
api.add_argument(
"--allowed-local-media-path",
default="",
help=("Root directory allowed for local media files. Unset disables "
"local files"))
api.add_argument("--log-level",
choices=("debug", "info", "warning", "error"),
default="info",
help="Server logging level (default: %(default)s)")
model = parser.add_argument_group("Model build and runtime")
model.add_argument(
"--cache-dir",
default="",
help=(
"Root for downloaded checkpoints and complete engine bundles. A "
"matching bundle is reused. A cache miss builds all components."),
)
model.add_argument(
"--engine-cache-max-size-gb",
type=_positive_float,
default=50.0,
help=
("Maximum disk space in GiB for compiled engine bundles. Older inactive "
"profiles are evicted least-recently-used."),
)
model.add_argument(
"--clear-engine-cache",
action="store_true",
help=("Remove compiled engine bundles before preparing this model. "
"Downloaded checkpoints are preserved."),
)
model.add_argument(
"--max-input-len",
type=_positive_int,
default=4096,
help=("Maximum prompt tokens, including conversation history "
"(default: %(default)s)"))
model.add_argument(
"--max-batch-size",
type=_positive_int,
default=1,
help=("Maximum number of sequences in an engine batch (default: "
"%(default)s)"))
model.add_argument(
"--max-kv-cache-capacity",
type=_positive_int,
default=8192,
help=("Maximum total tokens per sequence, including prompt and "
"generated tokens. The KV cache pool holds --max-batch-size "
"times this many tokens (default: %(default)s)"))
model.add_argument(
"--max-spec-verify-tokens",
type=_positive_int,
help=("Maximum verification tokens compiled into the base engine; "
"also caps the jump-forward width (default: the builder "
"default)"))
model.add_argument(
"--max-spec-draft-tokens",
type=_positive_int,
help=("Maximum draft tokens compiled into the draft engine "
"(default: the builder default)"))
model.add_argument(
"--max-image-tokens",
type=_positive_int,
help="Maximum total visual tokens in the visual engine profile")
model.add_argument(
"--max-image-tokens-per-image",
type=_positive_int,
help="Maximum visual tokens per image in the visual engine profile")
model.add_argument(
"--draft-top-k",
type=_positive_int,
help=("Candidate tokens retained per draft step. Defaults depend on "
"the speculative method"))
model.add_argument("--spec-draft-step", type=_positive_int)
model.add_argument("--spec-proposal-tokens", type=_positive_int)
model.add_argument("--spec-verify-tokens", type=_positive_int)
model.add_argument(
"--speculative-config",
default="",
help=(
"Speculative decoding JSON, e.g. "
"{\"method\":\"mtp\",\"num_speculative_tokens\":3}. Paired drafts "
"also use \"model\""))
model.add_argument(
"--enable-context-reuse",
action="store_true",
help=("Reuse matching text prefixes across requests in this trusted "
"server instance."),
)
model.add_argument(
"--enable-in-flight-batching",
action="store_true",
help="Admit queued requests into the running batch at generation "
"boundaries instead of serving one request at a time. Text, "
"multimodal-input and speculative-decoding deployments, with or "
"without context reuse: a standalone TTS deployment refuses the flag "
"at startup, and a Qwen3-Omni bundle serves text only (its speech "
"endpoints refuse).",
)
model.add_argument(
"--ifb-scheduling-mode",
default="auto",
help="With --enable-in-flight-batching, one of auto (default), mixed, "
"prefillFirst or singleCohort: mixed runs decode and chunked "
"prefill in one step and needs an engine whose layers support "
"compact execution; prefillFirst lets a pending prefill run alone; "
"auto picks mixed when supported and logs why when it does not; "
"singleCohort never runs compact steps and reproduces the outputs "
"of in-flight batching without them.",
)
model.add_argument(
"--max-num-tokens",
type=_non_negative_int,
default=0,
help="Token budget of a mixed step with decoding requests; 0 derives "
"it from the engine's batch size and input length.",
)
model.add_argument(
"--max-chunk-tokens",
type=_non_negative_int,
default=0,
help="Per-request prefill budget of a mixed step with decoding "
"requests, a multiple of the KV page size (128 tokens); 0 derives it "
"from --max-num-tokens. "
"Capped at --max-input-len, the most one prompt chunk holds.",
)
model.add_argument(
"--scheduling-policy",
default="fcfs",
help="Admission order of queued requests: arrival (fcfs), or the "
"request's priority field first, lower values first (priority).",
)
model.add_argument(
"--preemption",
default="none",
help="none (default) or pause: with pause, lower-priority requests "
"pause while a higher-priority "
"prompt is prefilled. Requires --scheduling-policy priority and the "
"mixed scheduling mode.",
)
model.add_argument(
"--priority-reserved-slots",
type=_non_negative_int,
default=0,
help="Batch slots kept for requests with a negative priority; other "
"requests are admitted only while more slots than this are free. "
"Requires --scheduling-policy priority. HTTP clients reach these "
"slots only when --min-request-priority is below 0 (the default 0 "
"refuses negative priorities; the server warns at startup).",
)
model.add_argument(
"--max-pause-steps",
type=_non_negative_int,
default=0,
help="With --preemption pause: the most steps in a row a request "
"stays paused; it is then not paused for one step before it can be "
"paused again. A generating request then produces a token; one whose "
"prompt is still being processed may get no prompt budget in that "
"step. 0 (default) for no limit, so sustained urgent traffic can keep "
"less urgent requests paused.",
)
model.add_argument("--ple-residency",
choices=("resident", "cached"),
default="resident")
model.add_argument(
"--ple-cache-slots",
type=_non_negative_int,
default=0,
help=
"Cached PLE pool slots, including reserved slot 0; selected at runtime. "
"Distinct valid rows across each execution batch must fit in slots - 1; "
"increase slots or reduce the execution chunk/batch if needed. "
"Smaller I/O batches do not relax this requirement.")
model.add_argument(
"--ple-file",
default="",
help=
"Original checkpoint file containing PLE; overrides the checkpoint source."
)
model.add_argument(
"--context-cache-max-records",
type=_non_negative_int,
default=1024,
help="Maximum retained context-cache records (default: %(default)s)")
model.add_argument(
"--context-cache-max-snapshots",
type=_non_negative_int,
default=0,
help=("Total snapshots kept for context reuse on a model with "
"recurrent layers (default: 4 x max batch size)"))
return parser
def parse_server_config(argv: Optional[Sequence[str]] = None,
*,
prog="tensorrt-edgellm-serve") -> ServerConfig:
parser = create_argument_parser(prog=prog)
args = parser.parse_args(argv)
if args.max_queued_requests < 0:
raise ServerConfigError("--max-queued-requests must be non-negative")
if args.queue_timeout <= 0:
raise ServerConfigError("--queue-timeout must be positive")
speculative = SpeculativeConfig.parse(args.speculative_config)
draft_step = args.spec_draft_step
proposal_tokens = args.spec_proposal_tokens
if (speculative and speculative.method in {"eagle3", "mtp"}
and speculative.num_speculative_tokens is not None):
draft_step = speculative.num_speculative_tokens
elif speculative and speculative.num_speculative_tokens is not None:
proposal_tokens = speculative.num_speculative_tokens
context_cache = ContextCacheConfig(
enabled=args.enable_context_reuse,
max_records=args.context_cache_max_records,
max_snapshots=args.context_cache_max_snapshots,
)
model = ModelConfig(
model=args.model,
cache_dir=args.cache_dir,
engine_cache_max_size_gb=args.engine_cache_max_size_gb,
clear_engine_cache=args.clear_engine_cache,
max_input_len=args.max_input_len,
max_batch_size=args.max_batch_size,
max_kv_cache_capacity=args.max_kv_cache_capacity,
max_spec_verify_tokens=args.max_spec_verify_tokens,
max_spec_draft_tokens=args.max_spec_draft_tokens,
max_image_tokens=args.max_image_tokens,
max_image_tokens_per_image=args.max_image_tokens_per_image,
draft_top_k=(speculative.draft_top_k
if speculative and speculative.draft_top_k is not None
else args.draft_top_k),
draft_step=draft_step,
proposal_tokens=proposal_tokens,
verify_tokens=(speculative.verify_tokens
if speculative and speculative.verify_tokens is not None
else args.spec_verify_tokens),
speculative_config=speculative,
context_cache_config=context_cache,
ple_config=PleRuntimeConfig(residency=args.ple_residency,
cache_slots=args.ple_cache_slots,
file_path=args.ple_file),
enable_in_flight_batching=args.enable_in_flight_batching,
ifb_scheduling_mode=args.ifb_scheduling_mode,
max_num_tokens=args.max_num_tokens,
max_chunk_tokens=args.max_chunk_tokens,
scheduling_policy=args.scheduling_policy,
preemption=args.preemption,
priority_reserved_slots=args.priority_reserved_slots,
max_pause_steps=args.max_pause_steps,
)
api = ApiConfig(
host=args.host,
port=args.port,
served_model_name=args.served_model_name,
api_key=args.api_key,
reasoning_parser=args.reasoning_parser,
tool_call_parser=args.tool_call_parser,
enable_auto_tool_choice=args.enable_auto_tool_choice,
max_queued_requests=args.max_queued_requests,
queue_timeout=args.queue_timeout,
allowed_local_media_path=args.allowed_local_media_path,
log_level=args.log_level,
min_request_priority=args.min_request_priority,
)
return ServerConfig(model=model, api=api)