{ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0. Generated by scripts/generate_trtllm_serve_schemas.py; do not edit by hand.", "$defs": { "AdvancedSamplingMode": { "description": "Deploy-time specialization of the one-model advanced sampler.\n\nFULL - per-row tensor top_k/top_p/min_p in one fused kernel (default; mixed\n per-request sampling). The only mode that accepts min_p.\nNO_TOPK - top_k disabled, top_p honored. Skips the top_k mask kernel.\nNO_TOPP - top_p disabled, top_k honored. Skips the top_p renorm kernel.\nNO_TOPK_NO_TOPP - both disabled (pure temperature sampling). Skips both kernels.", "enum": [ "full", "no_topk", "no_topp", "no_topk_no_topp" ], "title": "AdvancedSamplingMode", "type": "string" }, "AgentTreeConfig": { "additionalProperties": false, "description": "Configuration for agent tree scheduling.\n\nControls how agent requests are scheduled relative to regular chat requests.", "properties": { "agent_inflight_seq_num": { "default": 2147483647, "description": "Max number of inflight sequences for agent requests.", "title": "Agent Inflight Seq Num", "type": "integer" }, "agent_percentage": { "default": 0.0, "description": "The percentage of agent requests to schedule. Defaults to 0.0. Should be between 0.0 and 1.0. -1.0 means random schedule between agent and chatbot.", "title": "Agent Percentage", "type": "number" }, "agent_types": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Types of agents to schedule (e.g. 'AgentDeepResearch', 'Researcher', 'MultiroundChat').", "title": "Agent Types" } }, "title": "AgentTreeConfig", "type": "object" }, "AttentionDpConfig": { "additionalProperties": false, "description": "Configuration for attention DP.", "properties": { "batching_wait_iters": { "default": 10, "description": "The number of iterations to wait for batching.", "title": "Batching Wait Iters", "type": "integer" }, "enable_balance": { "default": false, "description": "Whether to enable balance.", "title": "Enable Balance", "type": "boolean" }, "enable_kv_cache_aware_routing": { "default": false, "description": "Enable internal KV cache-aware routing for attention DP. When enabled, distributes requests among ranks within a single instance's attention DP group, routing them to the rank with the matching prefix KV cache to reduce redundant prefill computation.", "title": "Enable Kv Cache Aware Routing", "type": "boolean" }, "kv_cache_routing_account_for_in_transfer": { "default": false, "description": "In-transfer load accounting in KV cache-aware routing. When True, requests still streaming KV to the GEN worker (tracked by the PyExecutor AsyncTransferManager but no longer in active_requests) are folded back into the per-rank load reported via RankState. This can improve balance under heavy disagg traffic but inflates num_active_requests reported upstream, which lets the inference loop's idle-fetch wait expire even when no requests are runnable and causes empty fetch cycles. Default False preserves the prior behaviour (fetch blocks when truly idle). Only used when enable_kv_cache_aware_routing is True.", "title": "Kv Cache Routing Account For In Transfer", "type": "boolean" }, "kv_cache_routing_cold_start_warmup": { "default": false, "description": "Cold-start mitigation in KV cache-aware routing. When True, the first tp_size relaxed requests after router init are round-robined across ranks (bypassing cache-affinity scoring) so every rank caches the shared system prompt before scoring would otherwise pin all traffic to the first warm rank. Only useful when requests share a long system prefix; for diverse-prompt workloads this can scatter requests that would otherwise consolidate on a single warm rank, wasting prefill. Default False preserves pre-warmup routing. Only used when enable_kv_cache_aware_routing is True.", "title": "Kv Cache Routing Cold Start Warmup", "type": "boolean" }, "kv_cache_routing_conversation_affinity": { "default": false, "description": "Enable explicit conversation-affinity routing for attention DP. When True, the first request is placed by kv_cache_routing_new_conv_placement and every subsequent request carrying the same conversation_params.conversation_id is pinned to that conversation's first-turn rank. OpenAI requests use the body conversation_params as canonical; the serve edge only creates conversation_params from the X-Session-ID header when the body does not provide it. This keeps a multi-turn conversation's KV-cache prefix on one rank (maximizing block reuse, minimizing cross-rank migration). Unlike enable_kv_cache_aware_routing (affinity inferred from prefix-match length, which is lost when blocks are evicted), the conversation->rank map is explicit and survives eviction. The same placement policy is used when no conversation_id is available. Takes precedence over enable_kv_cache_aware_routing when both are set.", "title": "Kv Cache Routing Conversation Affinity", "type": "boolean" }, "kv_cache_routing_fair_share_multiplier": { "default": 2.0, "description": "Loose per-rank active-request cap in KV cache-aware routing, expressed as a multiplier of the ceil fair-share (ceil((total_active + new) / tp_size)). Once a rank hits this cap within a scheduling batch it is removed from the eligible set for the remainder of the batch. Default 2.0 permits a 2x slack so cache affinity can dominate while preventing runaway concentration on a single rank. Set to 1.0 for strict fair share. Only used when enable_kv_cache_aware_routing is True.", "title": "Kv Cache Routing Fair Share Multiplier", "type": "number" }, "kv_cache_routing_load_balance_weight": { "default": 1.0, "description": "Weight (beta) for the load-balance term in KV cache-aware routing. Higher values prioritize load balance over cache affinity. Only used when enable_kv_cache_aware_routing is True.", "title": "Kv Cache Routing Load Balance Weight", "type": "number" }, "kv_cache_routing_match_rate_threshold": { "default": 0.1, "description": "Cache-affinity gate in KV cache-aware routing. For each request, match_len contributes to scoring only when max(match_len) / request_tokens across eligible ranks is strictly above this threshold; otherwise match_len is forced to 0 so routing is driven purely by load. Default 0.1 requires at least a 10% hit rate before cache affinity kicks in, which prevents a small universal prefix (e.g. a shared system prompt) from pinning all traffic to the first warm ranks. Set to 0.0 to honour any nonzero match. Only used when enable_kv_cache_aware_routing is True.", "title": "Kv Cache Routing Match Rate Threshold", "type": "number" }, "kv_cache_routing_max_sessions": { "default": 65536, "description": "LRU cap on the conversation->rank map used by conversation-affinity routing. The oldest conversations are evicted once more than this many are tracked, bounding memory on long-running servers. Only used when kv_cache_routing_conversation_affinity is True.", "title": "Kv Cache Routing Max Sessions", "type": "integer" }, "kv_cache_routing_new_conv_placement": { "default": "round_robin", "description": "Placement policy in conversation-affinity routing for requests with no pinned rank yet (first turn of a conversation, requests without a conversation_id, sticky overflow). 'round_robin' (default) rotates across eligible ranks. 'least_queued' chooses the fewest live requests. 'least_tokens' chooses the fewest active prompt tokens, including requests assigned in the current batch; ties use live request count, then the rotating rank cursor. Active prompt tokens estimate work, not cached KV residency. Existing conversation->rank pins are unaffected. Only used when kv_cache_routing_conversation_affinity is True.", "enum": [ "round_robin", "least_queued", "least_tokens" ], "title": "Kv Cache Routing New Conv Placement", "type": "string" }, "timeout_iters": { "default": 50, "description": "The number of iterations to timeout.", "title": "Timeout Iters", "type": "integer" } }, "title": "AttentionDpConfig", "type": "object" }, "AutoDecodingConfig": { "additionalProperties": false, "description": "Configuration for auto speculative decoding.\n\nThis config will automatically select a good, draft-model free\nspeculation algorithm with some heuristic.\n\nAttributes that are inherited from the base class are ignored.", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "AUTO", "default": "AUTO", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "AutoDecodingConfig", "type": "object" }, "BlockReuseConfig": { "additionalProperties": false, "description": "Configuration for KV cache block reuse policies.", "properties": { "max_num_turns": { "default": 1, "description": "Maximum number of completed conversation turns whose committed SWA-window blocks and Mamba stable-boundary state are retained by KV cache manager v2. Only used when `policy` is 'per_conversation'.", "exclusiveMinimum": 0, "status": "prototype", "title": "Max Num Turns", "type": "integer" }, "policy": { "default": "all_reusable", "description": "KV cache manager v2 block reuse policy. 'all_reusable' commits reusable blocks after every context chunk; 'per_request' commits them only after the final context chunk; 'per_conversation' uses 'per_request' commits and retains committed SWA-window blocks and Mamba stable-boundary state for up to `max_num_turns` completed turns. Periodic Mamba state snapshots are disabled with 'per_conversation'. All reusable blocks remain subject to normal cache eviction. Requests without conversation params use 'per_request' behavior. When 'all_reusable' and SWA scratch reuse are both enabled, only non-scratch blocks are committed for reuse.", "enum": [ "all_reusable", "per_request", "per_conversation" ], "status": "prototype", "title": "Policy", "type": "string" }, "swa_endpoint_rewind_tokens": { "default": 0, "description": "Extra tokens before the final SWA window whose cache blocks receive higher eviction priority, together with sink blocks. Zero disables the entire endpoint-priority callback. Positive values require KV cache manager v2, block reuse enabled, and policy='all_reusable'. This preference does not guarantee residency or change attention windows or prefix matching. Dummy and draft requests are excluded.", "minimum": 0, "status": "prototype", "title": "Swa Endpoint Rewind Tokens", "type": "integer" } }, "title": "BlockReuseConfig", "type": "object" }, "CSA2SparseAttentionConfig": { "additionalProperties": false, "description": "Use checkpoint-owned CSA2 geometry with the PyTorch sparse backend.", "properties": { "algorithm": { "const": "csa2", "default": "csa2", "title": "Algorithm", "type": "string" }, "use_fp8_staging": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "Stage CSA2 KV as E4M3 for native trtllm-gen on SM100-family. The per-tensor staging scale is a power of two derived every forward from the persistent pools' own group scales, so staged rows keep the E4M3 range they decode to. Left unset, staging turns on wherever that path is available: attn_backend=TRTLLM on an SM100-family GPU, an FP8 kv_cache_config.dtype, and no packed sparse attention. Set True to require it, which rejects a configuration that cannot serve it instead of falling back, or False to pin BF16 staging.", "status": "prototype", "title": "Use Fp8 Staging" } }, "title": "CSA2SparseAttentionConfig", "type": "object" }, "CacheTransceiverConfig": { "additionalProperties": false, "description": "Configuration for the cache transceiver.", "properties": { "agent_bounce_buffer_enable": { "default": false, "description": "Run the KV-cache bounce in the C++ transfer agent instead of the Python transceiver, using one shared kv_cache_bounce_size_mb buffer (use a power of two). Set this identically on context and generation; kv_cache_bounce_size_mb matters only when its usable capacity (rounded down to a power of two) is below max_chunk_size; both sides must then clamp to the same chunk cap, otherwise the pair falls back to standard NIXL. By default only writes with many small descriptors (>= 1024, <= 16 KiB average, i.e. head-mismatch layouts) take the path; head-matched layouts (MLA, symmetric TP) stay on standard NIXL unless agent_bounce_params relaxes the gate. Requires the Python (v2) transceiver; see the disaggregated-serving docs.", "title": "Agent Bounce Buffer Enable", "type": "boolean" }, "agent_bounce_params": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Expert tuning knobs for the C++ transfer-agent bounce pipeline; see tensorrt_llm/_torch/disaggregation/nixl/bounce_knobs.py for the valid keys. Byte-valued knobs accept a KB/MB/GB suffix (e.g. '32MB'). Precedence: this dict > environment variable > built-in default. Requires agent_bounce_buffer_enable. request_timeout_ms and the effective max_chunk_size (after the arena clamp) must match between context and generation. max_average_descriptor_size=0 stops outbound routing only (arena, handshake and inbound scatter stay active); set agent_bounce_buffer_enable=False to turn the feature off.", "title": "Agent Bounce Params" }, "backend": { "anyOf": [ { "enum": [ "DEFAULT", "UCX", "NIXL", "MOONCAKE", "MPI" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "The communication backend type to use for the cache transceiver.", "title": "Backend" }, "enable_pipelined_transfer": { "default": false, "description": "Transfer each completed prefill chunk's KV cache while later chunks compute. Requires Python NIXL, generation-first scheduling, chunked prefill, pipeline_parallel_size=1, context_parallel_size=1 on both peers, beam_width=1, no Python bounce buffer (the C++ transfer-agent bounce selected by agent_bounce_buffer_enable is allowed) or Mamba/hybrid cache, and block reuse disabled or set to all_reusable. Invalid static settings fail at startup; per-request constraints reject the request.", "title": "Enable Pipelined Transfer", "type": "boolean" }, "kv_cache_bounce_size_mb": { "default": 0, "description": "Capacity in MiB of the native-disagg KV-cache bounce buffer, which coalesces a request's scattered per-block KV for a single multi-rail NIXL write. The size doubles as the on/off switch: 0 (default) keeps the per-block path, >0 enables bounce at that capacity. By default two buffers of this size are allocated (one for send, one for recv); with agent_bounce_buffer_enable a single shared buffer is allocated instead and the size should be a power of two (256/512/1024). Requires the Python (v2) transceiver (transceiver_runtime); the C++ transceiver does not support bounce and ignores this field.", "minimum": 0, "title": "Kv Cache Bounce Size Mb", "type": "integer" }, "kv_transfer_poll_interval_ms": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": 5000, "description": "Bounded wait interval in milliseconds for polling KV transfer progress when active transfers block disaggregated admission.", "title": "Kv Transfer Poll Interval Ms" }, "kv_transfer_sender_future_timeout_ms": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": 1000, "description": "Duration in milliseconds of each bounded sender future wait slice while polling KV transfer completion. It does not set the overall transfer deadline.", "title": "Kv Transfer Sender Future Timeout Ms" }, "kv_transfer_timeout_ms": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": 60000, "description": "KV cache transfer timeout in milliseconds. Blocking sender waits use it as an absolute deadline; blocking receive task waits use it per task. The Python V2 transceiver requires a finite value; None remains available to other runtimes. It is distinct from the sender future wait slice.", "title": "Kv Transfer Timeout Ms" }, "max_tokens_in_buffer": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The max number of tokens the transfer buffer can fit.", "title": "Max Tokens In Buffer" }, "transceiver_runtime": { "anyOf": [ { "enum": [ "CPP", "PYTHON", "auto" ], "type": "string" }, { "type": "null" } ], "default": "PYTHON", "description": "The runtime implementation. 'PYTHON' (default) selects the Python transceiver, while 'CPP' selects the C++ transceiver and requires kv_cache_config.use_kv_cache_manager_v2=False with a model that supports V1. 'auto' adopts the model's preferred runtime when it declares one; otherwise it selects the Python transceiver, falling back to the C++ transceiver only when this config itself rules it out (non-NIXL backend or a null kv_transfer_timeout_ms) \u2014 any other incompatibility fails at transceiver creation. The fallback is decided independently on each server and is only logged, not surfaced, so keep context and generation server configurations consistent. None is equivalent to 'CPP'. 'auto' is resolved on the PyTorch backend's standard model-loading path.", "title": "Transceiver Runtime" } }, "title": "CacheTransceiverConfig", "type": "object" }, "CapacitySchedulerPolicy": { "enum": [ "MAX_UTILIZATION", "GUARANTEED_NO_EVICT", "STATIC_BATCH" ], "title": "CapacitySchedulerPolicy", "type": "string" }, "ColdPageQuantizationCompressionConfig": { "additionalProperties": false, "description": "Quantize Host and Disk KV pages without changing the active GPU cache.", "properties": { "algorithm": { "const": "quantization_for_cold_page", "default": "quantization_for_cold_page", "title": "Algorithm", "type": "string" }, "nvfp4_residual_dim": { "default": 64, "description": "64 (default): use two independently scaled FP4 components for the 64 DeepSeek-V4 target cold-page RoPE values: a main component and its residual. 0: use a single NVFP4 component. Only 0 and 64 are supported. Ignored for other models, draft caches, and when skip_rope_quantization preserves the original RoPE precision.", "enum": [ 0, 64 ], "status": "prototype", "title": "Nvfp4 Residual Dim", "type": "integer" }, "quant": { "const": "nvfp4", "default": "nvfp4", "description": "Quantization format stored in the compressed cache tier.", "title": "Quant", "type": "string" }, "scale_checkpoint_path": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ], "default": null, "description": "Optional local ModelOpt NVFP4 checkpoint directory supplying per-layer K/V global scales. Omit it to use identity global scales.", "telemetry": { "exclude": true }, "title": "Scale Checkpoint Path" }, "skip_rope_quantization": { "default": false, "description": "True: preserve the position-encoded (RoPE) part of each K vector in its original active-cache precision and quantize the rest to NVFP4. False (default): quantize both parts, using residual RoPE quantization for DeepSeek-V4 when nvfp4_residual_dim is 64. An option to explore; measure its accuracy effect on your model.", "status": "prototype", "title": "Skip Rope Quantization", "type": "boolean" } }, "title": "ColdPageQuantizationCompressionConfig", "type": "object" }, "ContextChunkingPolicy": { "description": "Context chunking policy.", "enum": [ "FIRST_COME_FIRST_SERVED", "EQUAL_PROGRESS", "FORCE_CHUNK" ], "title": "ContextChunkingPolicy", "type": "string" }, "CpConfig": { "additionalProperties": false, "description": "Configuration for context parallelism.", "properties": { "cp_type": { "$ref": "#/$defs/CpType", "default": "ULYSSES", "description": "Context parallel type." }, "fifo_version": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "FIFO version for alltoall communication. Used in HELIX parallelism. Defaults to 2.", "title": "Fifo Version" }, "tokens_per_block": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Number of tokens per block. Used in HELIX parallelism.", "title": "Tokens Per Block" }, "use_nccl_for_alltoall": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "Whether to use NCCL for alltoall communication. Used in HELIX parallelism. Defaults to True.", "title": "Use Nccl For Alltoall" } }, "title": "CpConfig", "type": "object" }, "CpType": { "enum": [ "ULYSSES", "RING", "HELIX" ], "title": "CpType", "type": "string" }, "DFlashDecodingConfig": { "additionalProperties": false, "description": "Configuration for DFlash speculative decoding.\n\nDFlash is a target-dependent speculative decoding method that uses\nhidden states from specific target model layers as cross-attention\ncontext in the draft model to predict multiple draft tokens in parallel.\n\nKey features:\n- Target-dependent: uses hidden states from target model layers\n- Parallel prediction: all K draft tokens in one forward pass\n- Cross-attention: draft model attends to target hidden states\n\nReference: https://arxiv.org/pdf/2602.06036", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "attention_backend": { "default": "VANILLA", "description": "Attention backend for DFlash pooled-context cross-attention, independent of the drafter's standard attention modules. VANILLA uses FlashAttention with a contiguous context K/V cache and runs anywhere. TRTLLM uses TRTLLM-Gen FMHA (via FlashInfer) over a private paged context K/V cache and supports SM100/SM103 only. FA4 uses the flash-attn CuTe DSL kernels on the same paged cache and supports SM90 only.", "enum": [ "VANILLA", "TRTLLM", "FA4" ], "title": "Attention Backend", "type": "string" }, "context_recompute_tail": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 0, "description": "Number of prompt-tail tokens to recompute through the target forward when a request takes a KV-cache prefix hit, so the drafter's hidden-state context covers them (reused tokens never pass a target forward, which otherwise degrades acceptance length exactly when prefix caching helps most). 0 (the default) disables the recompute: the feature is opt-in, because recomputing reused tokens trades TTFT and prefill throughput for acceptance length and disables KV-connector prefix loads for the recomputed span. None resolves from the draft model config: dflash_config.swa_window_size when the drafter's context attention is windowed (a tail of the window size reproduces the no-reuse drafter inputs exactly), else -1. -1 forces a full re-prefill on a hit. Prefix reuse ahead of the recomputed tail is kept. Requires chunked prefill and the all_reusable block-reuse policy, and sliding-window attention layers are unsupported on the V1 KV cache manager; the KV cache managers disable it with a warning otherwise.", "title": "Context Recompute Tail" }, "decoding_type": { "const": "DFlash", "default": "DFlash", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "mask_token_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The token ID used as a mask token for parallel draft prediction. If None, it will be read from the draft model config (dflash_config.mask_token_id).", "title": "Mask Token Id" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "skip_ctx_buffer_budget_check": { "default": false, "description": "Skip the config-time check that the pooled-context K/V buffers fit in the memory left free after the KV-cache pool commits. The estimate is deliberately conservative; set this to proceed when you know the device has the headroom. The token-budget check is not affected.", "title": "Skip Ctx Buffer Budget Check", "type": "boolean" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "target_layer_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of target model layer indices whose hidden states are captured for cross-attention in the draft model. If None, read from the draft model config (dflash_config.target_layer_ids).", "title": "Target Layer Ids" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "DFlashDecodingConfig", "type": "object" }, "DSparkDecodingConfig": { "additionalProperties": false, "description": "Configuration for DSpark speculative decoding.\n\nDSpark (DeepSeek) is a target-dependent, \"semi-parallel\" speculative\ndecoding method. Like DFlash it captures hidden states from several target\nlayers as cross-attention context and drafts a whole block in a single\nbackbone forward, but it additionally refines the per-position draft logits\nwith a lightweight sequential head (a low-rank Markov head, optionally an RNN\nhead). A confidence head can score each draft position for verification\nscheduling without truncating the proposed block. This configuration selects\nnative full-block drafting and does not expose confidence-scheduling or\ncalibration controls.\n\nKey features:\n- Target-dependent: captures hidden states from ``target_layer_ids``.\n- Semi-parallel: one block backbone forward + cheap sequential head refine.\n- Full-block proposals: confidence scoring does not change the draft tokens\n or the standard target-verification acceptance rule.\n\nReference: DeepSeek DeepSpec (https://github.com/deepseek-ai/DeepSpec).", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "attention_backend": { "default": "AUTO", "description": "Block-decode attention backend for a standalone DSpark drafter (one shipped as its own checkpoint, not inside the target's mtp.* namespace). Ignored by the embedded DeepSeek-V4-Pro draft. Independent of the backend that builds the drafter's own attention modules.\n\nAUTO resolves per drafter family and is right unless you are pinning a kernel: TRTLLM for a GQA backbone, which degrades when its kernel is missing, and CUTEDSL for an MLA one, which raises. TRTLLM needs FlashInfer and SM100/SM103. CUTEDSL is MLA-only: one cute-dsl MLA decode pass over context and block, on SM100/SM103/SM107. Which kernel each name selects: MLADSparkForCausalLM.", "enum": [ "AUTO", "VANILLA", "TRTLLM", "CUTEDSL" ], "title": "Attention Backend", "type": "string" }, "block_size": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "Number of draft positions produced per block. If None, read from the draft model config (dspark_block_size). Should equal max_draft_len.", "title": "Block Size" }, "decoding_type": { "const": "DSpark", "default": "DSpark", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "markov_head_type": { "anyOf": [ { "enum": [ "vanilla", "gated", "rnn" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Type of the sequential refinement head used within a block. If None, read from the draft model config (dspark_markov_head_type), defaulting to \"vanilla\".", "title": "Markov Head Type" }, "markov_rank": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Low-rank dimension of the Markov head logit-bias. If None, read from the draft model config (dspark_markov_rank). 0 disables the head.", "title": "Markov Rank" }, "mask_token_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Token ID used as the mask/noise token for parallel draft prediction. If None, read from the draft model config (dspark_noise_token_id).", "title": "Mask Token Id" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "target_layer_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Target model layer indices whose hidden states are captured for cross-attention in the draft model. If None, read from the draft model config (dspark_target_layer_ids).", "title": "Target Layer Ids" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "DSparkDecodingConfig", "type": "object" }, "DecodeCudaGraphConfig": { "additionalProperties": false, "description": "CUDA graph configuration for decode requests.", "properties": { "batch_sizes": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of batch sizes to create CUDA graphs for.", "title": "Batch Sizes" }, "enable_padding": { "default": false, "description": "If true, batches are rounded up to the nearest cuda_graph_batch_size. This is usually a net win for performance.", "title": "Enable Padding", "type": "boolean" }, "max_batch_size": { "default": 0, "description": "Maximum batch size for CUDA graphs.", "minimum": 0, "title": "Max Batch Size", "type": "integer" }, "mode": { "const": "decode", "default": "decode", "description": "CUDA graph configuration mode.", "title": "Mode", "type": "string" } }, "title": "DecodeCudaGraphConfig", "type": "object" }, "DeepSeekSparseAttentionConfig": { "additionalProperties": false, "description": "Configuration for DeepSeek Sparse Attention.", "properties": { "algorithm": { "const": "dsa", "default": "dsa", "title": "Algorithm", "type": "string" }, "enable_heuristic_topk": { "default": false, "description": "Whether to enable Guess-Verify-Refine (GVR) Top-K for the DSA decode indexer instead of the exact insertion/radix Top-K path. Currently supported for index_topk \u2208 {512, 1024, 2048} on Blackwell (SM100+), with compress_ratio \u2208 {1, 4} (DSv3.2 + DSv4 indexers). Falls back to the production insertion/radix Top-K path when prerequisites are not met. `use_self_sampling_topk` selects the GVR engine generation.", "title": "Enable Heuristic Topk", "type": "boolean" }, "gvr_deterministic_topk": { "default": false, "description": "Make the self-sampling GVR Top-K deterministic: the selected index set is {score > tau} plus the smallest indices among the ties at tau, and the output order is a fixed function of the input (decode and prefill). Costs about 0.2-3 us per Top-K launch. Only takes effect with enable_heuristic_topk=True and use_self_sampling_topk=True; ignored otherwise.", "title": "Gvr Deterministic Topk", "type": "boolean" }, "index_head_dim": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The dimension of the indexer heads.", "title": "Index Head Dim" }, "index_n_heads": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of heads for the indexer.", "title": "Index N Heads" }, "index_share_for_mtp_iteration": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "Reuse the indexer Top-K across MTP draft steps instead of recomputing it each step. Defaults to the model's HF config value.", "status": "prototype", "title": "Index Share For Mtp Iteration" }, "index_topk": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The topk for the indexer.", "title": "Index Topk" }, "indexer_k_dtype": { "default": "fp8", "description": "Data type used for the indexer K cache. `fp8` stores one FP8 E4M3 byte per element with a per-128 float32 scale; `fp4` packs two FP4 E2M1 codes per byte with a per-32 UE8M0 exponent, halving the per-token indexer K footprint (132 B to 68 B at index_head_dim=128). `fp4` requires Blackwell+ (SM>=100) at runtime and index_head_dim=128.", "enum": [ "fp8", "fp4" ], "title": "Indexer K Dtype", "type": "string" }, "indexer_max_chunk_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum chunk size for the indexer.", "title": "Indexer Max Chunk Size" }, "indexer_rope_interleave": { "default": false, "description": "Whether to use interleaved RoPE layout for the indexer.", "title": "Indexer Rope Interleave", "type": "boolean" }, "q_split_threshold": { "default": 8192, "description": "If number of packed tokens in prefill chunk exceeds this threshold, q tokens will be evenly distributed across ranks for indexer computation. If negative, q split will always be disabled.", "title": "Q Split Threshold", "type": "integer" }, "seq_len_threshold": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The sequence length threshold for separating short and long sequences.", "title": "Seq Len Threshold" }, "skip_indexer_for_short_seqs": { "default": true, "description": "Whether to skip the MQA and Top-K in the indexer for short sequences.", "title": "Skip Indexer For Short Seqs", "type": "boolean" }, "use_cute_dsl_paged_mqa_logits": { "default": false, "description": "Whether to use CuTE DSL paged MQA logits kernel on SM100-family GPUs instead of C++ DeepGEMM.", "title": "Use Cute Dsl Paged Mqa Logits", "type": "boolean" }, "use_cute_dsl_topk": { "default": false, "description": "Whether to use CuTE DSL top-k kernel instead of the CUDA C++ indexer_topk_decode.", "title": "Use Cute Dsl Topk", "type": "boolean" }, "use_gvr_emission": { "default": false, "description": "Enable the emission-assisted block-skip optimization for the temporal-hint GVR engine. When set, the FP4 indexer epilogue emits per-block max logits so the GVR Top-K can skip whole blocks. Only takes effect with enable_heuristic_topk=True, use_self_sampling_topk=False, and the FP4 paged-MQA-logits path; ignored otherwise. The self-sampling engine derives its bracket from the current row and does not use emission.", "title": "Use Gvr Emission", "type": "boolean" }, "use_self_sampling_topk": { "default": true, "description": "Select the GVR engine generation when enable_heuristic_topk is set: True (default) runs the hint-free self-sampling engine, which derives its search bracket from the current row and keeps no cross-step state; False runs the temporal-hint engines, which reuse the previous decode step's Top-K indices as hints. Ignored when enable_heuristic_topk is False.", "title": "Use Self Sampling Topk", "type": "boolean" } }, "title": "DeepSeekSparseAttentionConfig", "type": "object" }, "DeepSeekV4SparseAttentionConfig": { "additionalProperties": false, "description": "Configuration for DeepSeek-V4 Sparse Attention.", "properties": { "algorithm": { "const": "deepseek_v4", "default": "deepseek_v4", "title": "Algorithm", "type": "string" }, "compress_ratios": { "description": "The compress ratios of each layer. DeepSeek-V4 uses 0 for uncompressed/SWA-only layers; the LLM API config normalizes 0 to 1, while checkpoint-facing semantics remain unchanged.", "items": { "type": "integer" }, "title": "Compress Ratios", "type": "array" }, "enable_heuristic_topk": { "default": false, "description": "Whether to enable Guess-Verify-Refine (GVR) Top-K for the DSA decode indexer instead of the exact insertion/radix Top-K path. Currently supported for index_topk \u2208 {512, 1024, 2048} on Blackwell (SM100+), with compress_ratio \u2208 {1, 4} (DSv3.2 + DSv4 indexers). Falls back to the production insertion/radix Top-K path when prerequisites are not met. `use_self_sampling_topk` selects the GVR engine generation.", "title": "Enable Heuristic Topk", "type": "boolean" }, "enable_kv_cache_offload": { "default": false, "description": "Offload ratio-4 compressed attention KV history to host memory with KV cache manager v2. The indexer and sliding-window caches remain on GPU. This feature is under development and currently cannot be enabled for inference.", "status": "prototype", "title": "Enable Kv Cache Offload", "type": "boolean" }, "gvr_deterministic_topk": { "default": false, "description": "Make the self-sampling GVR Top-K deterministic: the selected index set is {score > tau} plus the smallest indices among the ties at tau, and the output order is a fixed function of the input (decode and prefill). Costs about 0.2-3 us per Top-K launch. Only takes effect with enable_heuristic_topk=True and use_self_sampling_topk=True; ignored otherwise.", "title": "Gvr Deterministic Topk", "type": "boolean" }, "index_head_dim": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 128, "description": "The dimension of the DeepSeek-V4 indexer heads.", "title": "Index Head Dim" }, "index_n_heads": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of heads for the indexer.", "title": "Index N Heads" }, "index_share_for_mtp_iteration": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "Reuse the indexer Top-K across MTP draft steps instead of recomputing it each step. Defaults to the model's HF config value.", "status": "prototype", "title": "Index Share For Mtp Iteration" }, "index_topk": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 512, "description": "The top-k for the indexer.", "title": "Index Topk" }, "indexer_k_dtype": { "default": "fp4", "description": "Data type used for the indexer K cache. DeepSeek-V4 defaults to `fp4` to reduce the per-token indexer K footprint on Blackwell+ (SM>=100). Set to `fp8` for the legacy FP8 indexer K cache path.", "enum": [ "fp8", "fp4" ], "title": "Indexer K Dtype", "type": "string" }, "indexer_max_chunk_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum chunk size for the indexer.", "title": "Indexer Max Chunk Size" }, "indexer_rope_interleave": { "default": false, "description": "Whether to use interleaved RoPE layout for the indexer.", "title": "Indexer Rope Interleave", "type": "boolean" }, "q_split_threshold": { "default": 8192, "description": "If number of packed tokens in prefill chunk exceeds this threshold, q tokens will be evenly distributed across ranks for indexer computation. If negative, q split will always be disabled.", "title": "Q Split Threshold", "type": "integer" }, "seq_len_threshold": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The sequence length threshold for separating short and long sequences.", "title": "Seq Len Threshold" }, "skip_indexer_for_short_seqs": { "default": false, "description": "Whether to skip the MQA and Top-K in the indexer for short sequences.", "title": "Skip Indexer For Short Seqs", "type": "boolean" }, "use_cute_dsl_paged_mqa_logits": { "default": false, "description": "Whether to use CuTE DSL paged MQA logits kernel on SM100-family GPUs instead of C++ DeepGEMM.", "title": "Use Cute Dsl Paged Mqa Logits", "type": "boolean" }, "use_cute_dsl_topk": { "default": false, "description": "Whether to use CuTE DSL top-k kernel instead of the CUDA C++ indexer_topk_decode.", "title": "Use Cute Dsl Topk", "type": "boolean" }, "use_gvr_emission": { "default": false, "description": "Enable the emission-assisted block-skip optimization for the temporal-hint GVR engine. When set, the FP4 indexer epilogue emits per-block max logits so the GVR Top-K can skip whole blocks. Only takes effect with enable_heuristic_topk=True, use_self_sampling_topk=False, and the FP4 paged-MQA-logits path; ignored otherwise. The self-sampling engine derives its bracket from the current row and does not use emission.", "title": "Use Gvr Emission", "type": "boolean" }, "use_self_sampling_topk": { "default": true, "description": "Select the GVR engine generation when enable_heuristic_topk is set: True (default) runs the hint-free self-sampling engine, which derives its search bracket from the current row and keeps no cross-step state; False runs the temporal-hint engines, which reuse the previous decode step's Top-K indices as hints. Ignored when enable_heuristic_topk is False.", "title": "Use Self Sampling Topk", "type": "boolean" }, "window_size": { "default": 128, "description": "The sliding window size in tokens for SWA layers.", "title": "Window Size", "type": "integer" } }, "title": "DeepSeekV4SparseAttentionConfig", "type": "object" }, "DisaggServerBlock": { "additionalProperties": false, "properties": { "_mpi_session": { "description": "Live objects require the Python LLM API.", "type": "null" }, "allow_request_chat_template": { "default": false, "type": "boolean" }, "allreduce_strategy": { "anyOf": [ { "enum": [ "AUTO", "NCCL", "UB", "MINLATENCY", "ONESHOT", "TWOSHOT", "LOWPRECISION", "MNNVL", "NCCL_SYMMETRIC" ], "type": "string" }, { "type": "null" } ], "default": "AUTO", "description": "Allreduce strategy to use.", "status": "beta", "title": "Allreduce Strategy" }, "attention_dp_config": { "anyOf": [ { "$ref": "#/$defs/AttentionDpConfig" }, { "type": "null" } ], "default": null, "description": "Optimized load-balancing for the DP Attention scheduler.", "status": "beta" }, "attn_backend": { "default": "TRTLLM", "description": "Attention backend to use.", "status": "beta", "telemetry": { "allowed_values": [ "VANILLA", "TRTLLM", "FLASHINFER" ] }, "title": "Attn Backend", "type": "string" }, "backend": { "const": "pytorch", "default": "pytorch", "description": "The backend to use for this LLM instance.", "status": "deprecated", "title": "Backend", "type": "string" }, "batch_wait_max_tokens_ratio": { "default": 0, "description": "Token accumulation threshold ratio for batch scheduling optimization. If greater than 0, the scheduler will accumulate requests locally until the total token count reaches batch_wait_max_tokens_ratio * max_num_tokens. This mechanism enhances GPU utilization efficiency by ensuring adequate batch sizes. If 0, disables token-based batching delays.", "maximum": 1, "minimum": 0, "status": "prototype", "title": "Batch Wait Max Tokens Ratio", "type": "number" }, "batch_wait_timeout_iters": { "default": 0, "description": "Maximum number of iterations the scheduler will wait to accumulate new coming requests for improved GPU utilization efficiency. If greater than 0, the scheduler will delay batch processing to gather more requests up to the specified iteration limit. If 0, disables timeout-iters-based batching delays.", "minimum": 0, "status": "prototype", "title": "Batch Wait Timeout Iters", "type": "integer" }, "batch_wait_timeout_ms": { "default": 0, "description": "If greater than 0, the request queue might wait up to batch_wait_timeout_ms to receive max_batch_size requests, if fewer than max_batch_size requests are currently available. If 0, no waiting occurs.", "minimum": 0, "status": "prototype", "title": "Batch Wait Timeout Ms", "type": "number" }, "batched_logits_processor": { "description": "Live objects require the Python LLM API.", "type": "null" }, "bounded_replay_on_generation": { "default": false, "description": "Run the DeepSeek-V4.1 bounded decoder replay tail on the generation worker in disaggregated serving. Configure both worker roles alike.", "status": "prototype", "telemetry": { "exclude": true }, "title": "Bounded Replay On Generation", "type": "boolean" }, "cache_transceiver_config": { "anyOf": [ { "$ref": "#/$defs/CacheTransceiverConfig" }, { "type": "null" } ], "default": null, "description": "Cache transceiver config.", "status": "prototype" }, "checkpoint_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The registered checkpoint loader format to use. `MX` selects ModelExpress as an opportunistic P2P loading path and falls back to loading the provided Hugging Face checkpoint; it does not require converting that checkpoint to an MX-specific format. You may use a custom checkpoint format by subclassing `BaseCheckpointLoader` and registering it with `register_checkpoint_loader`.\nIf neither checkpoint_format nor checkpoint_loader are provided, checkpoint_format will be set to HF and the default HfCheckpointLoader will be used.\nIf checkpoint_format and checkpoint_loader are both provided, checkpoint_loader will be ignored.", "status": "prototype", "title": "Checkpoint Format" }, "checkpoint_io_policy": { "default": "auto", "description": "Controls checkpoint storage I/O independently of checkpoint format. 'auto' selects rank-striped read-ahead for compatible built-in PyTorch/HF loads and selects native I/O otherwise. 'native' preserves the existing loader. 'rank_striped_read_ahead' lets node-local ranks read disjoint SafeTensors extents while native mapping, materialization, and H2D continue. Incompatible configurations select native I/O before optimized reader or collective setup. Runtime-ineligible loads fall back to native before model mutation.", "enum": [ "auto", "native", "rank_striped_read_ahead" ], "status": "prototype", "title": "Checkpoint Io Policy", "type": "string" }, "checkpoint_loader": { "description": "Live objects require the Python LLM API.", "type": "null" }, "context_parallel_size": { "default": 1, "description": "The context parallel size.", "title": "Context Parallel Size", "type": "integer" }, "cp_config": { "anyOf": [ { "$ref": "#/$defs/CpConfig" }, { "type": "null" } ], "default": null, "description": "Context parallel config.", "status": "prototype" }, "cuda_graph_config": { "anyOf": [ { "$ref": "#/$defs/DecodeCudaGraphConfig" }, { "$ref": "#/$defs/EncodeCudaGraphConfig" }, { "type": "null" } ], "description": "CUDA graph config. If true, use CUDA graphs for decoding. CUDA graphs are only created for the batch sizes in cuda_graph_config.batch_sizes, and are enabled for batches that consist of decoding requests *only* (the reason is that it's hard to capture a single graph with prefill requests since the input shapes are a function of the sequence lengths). Note that each CUDA graph can use up to 200 MB of extra memory.", "status": "beta", "title": "Cuda Graph Config" }, "custom_tokenizer": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Specify a custom tokenizer implementation. Accepts either: (1) a built-in alias (e.g., 'deepseek_v32'), or (2) a Python import path (e.g., 'tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer'). The tokenizer class must implement 'from_pretrained(path, **kwargs)' and the TokenizerBase interface.", "status": "prototype", "title": "Custom Tokenizer" }, "decoding_config": { "type": [ "object", "null" ] }, "disable_mm_encoder": { "default": false, "description": "Skip instantiating and loading the multimodal (e.g. vision) encoder of a multimodal checkpoint and serve it text-only. Saves the encoder's GPU memory (enlarging the KV cache pool) for workloads that never send image/video/audio inputs; such requests are rejected. Only takes effect for model implementations that support it (currently Mistral3 and the Qwen3-VL / Qwen3.5-VL family); a no-op otherwise. Defaults to False.", "status": "prototype", "title": "Disable Mm Encoder", "type": "boolean" }, "disable_overlap_scheduler": { "default": false, "description": "Disable the overlap scheduler.", "status": "beta", "title": "Disable Overlap Scheduler", "type": "boolean" }, "disagg_cluster": { "anyOf": [ { "additionalProperties": false, "properties": { "cluster_name": { "default": "", "title": "Cluster Name", "type": "string" }, "cluster_uri": { "title": "Cluster Uri", "type": "string" }, "heartbeat_interval_sec": { "default": 5, "title": "Heartbeat Interval Sec", "type": "integer" }, "inactive_timeout_sec": { "default": 10, "title": "Inactive Timeout Sec", "type": "integer" }, "minimal_instances": { "anyOf": [ { "$ref": "#/$defs/MinimalInstances" }, { "type": "null" } ], "default": null } }, "required": [ "cluster_uri" ], "title": "DisaggClusterConfig", "type": "object" }, { "type": "null" } ] }, "dtype": { "default": "auto", "description": "The data type to use for the model. When 'auto' (default), it is read from the HF config.json ('dtype', or the deprecated 'torch_dtype'); for composite/VLM configs it falls back to the nested text_config.dtype. Defaults to bfloat16 if none is found.", "telemetry": { "allowed_values": [ "auto", "float16", "bfloat16", "float32" ] }, "title": "Dtype", "type": "string" }, "dwdp_config": { "anyOf": [ { "$ref": "#/$defs/DwdpConfig" }, { "type": "null" } ], "default": null, "description": "DWDP (Distributed Weight Data Parallelism) config.", "status": "prototype" }, "enable_attention_dp": { "default": false, "description": "Enable attention data parallel.", "status": "beta", "title": "Enable Attention Dp", "type": "boolean" }, "enable_autotuner": { "default": true, "description": "Enable autotuner for all tunable ops. This flag is for debugging purposes only, and the performance may significantly degrade if set to false.", "status": "prototype", "title": "Enable Autotuner", "type": "boolean" }, "enable_chunked_prefill": { "default": false, "description": "Enable chunked prefill.", "title": "Enable Chunked Prefill", "type": "boolean" }, "enable_early_first_token_response": { "default": false, "description": "Under the overlap scheduler, emit the first-token response ahead of the next sample step to reduce TTFT. No effect when the overlap scheduler is disabled.", "status": "prototype", "title": "Enable Early First Token Response", "type": "boolean" }, "enable_encoder_decoder_mixed_cuda_graph": { "default": true, "description": "Enable the mixed-batch CUDA graph performance optimization for encoder-decoder models. The graph handles decoder iterations containing both context and generation requests. It is enabled by default when both `cuda_graph_config` and `encoder_cuda_graph_config` produce usable graph shapes.", "status": "prototype", "title": "Enable Encoder Decoder Mixed Cuda Graph", "type": "boolean" }, "enable_energy_metrics": { "default": false, "description": "Enable GPU energy monitoring via NVML. When enabled, the server exposes an /energy_metrics endpoint that reports cumulative GPU energy consumption in joules.", "status": "prototype", "title": "Enable Energy Metrics", "type": "boolean" }, "enable_in_graph_sampling": { "default": false, "description": "Opt-in in-graph sampling: capture part of the sampling work into the model's CUDA graph and run it there, instead of eagerly after the forward. The sampler itself is unchanged; only where its work runs differs. Batches whose sampling cannot be captured are unaffected and keep the eager path. This captures an additional set of CUDA graphs at startup, which makes warmup noticeably slower and costs extra memory, in exchange for lower per-step sampling overhead.", "status": "prototype", "title": "Enable In Graph Sampling", "type": "boolean" }, "enable_iter_perf_stats": { "default": false, "description": "Enable iteration performance statistics.", "status": "prototype", "title": "Enable Iter Perf Stats", "type": "boolean" }, "enable_iter_req_stats": { "default": false, "description": "If true, enables per request stats per iteration. Must also set enable_iter_perf_stats to true to get request stats.", "status": "prototype", "title": "Enable Iter Req Stats", "type": "boolean" }, "enable_layerwise_nvtx_marker": { "default": false, "description": "If true, enable layerwise nvtx marker.", "status": "beta", "title": "Enable Layerwise Nvtx Marker", "type": "boolean" }, "enable_lm_head_tp_in_adp": { "default": false, "description": "Enable LM head TP in attention dp.", "status": "prototype", "title": "Enable Lm Head Tp In Adp", "type": "boolean" }, "enable_locality_domains": { "default": false, "description": "Enable locality domain execution for supported PyTorch backend ops.", "status": "prototype", "title": "Enable Locality Domains", "type": "boolean" }, "enable_lora": { "default": false, "description": "Enable LoRA.", "title": "Enable Lora", "type": "boolean" }, "enable_low_latency_host_dispatch": { "default": false, "description": "Use low-latency spin-wait mode for CUDA host task dispatch (cudaLaunchHostFunc_v2 with cudaHostTaskSpinWait). Reduces callback latency at the cost of a CPU core spinning while waiting for the GPU event. Requires CUDA 13.2+; on older CUDA versions, falls back to the default blocking mode and logs a one-time warning.", "status": "prototype", "title": "Enable Low Latency Host Dispatch", "type": "boolean" }, "enable_min_latency": { "default": false, "description": "If true, enable min-latency mode. Currently only used for Llama4.", "status": "beta", "title": "Enable Min Latency", "type": "boolean" }, "enable_mla_skip_correction": { "default": false, "description": "Enable threshold-based skip-correction for trtllm-gen MLA attention kernels on SM100 and SM103. When enabled, mla_skip_correction_threshold controls the optimization threshold.", "status": "prototype", "title": "Enable Mla Skip Correction", "type": "boolean" }, "enable_resource_governor": { "default": false, "description": "Enable the resource governor for runtime cache management operations such as KV cache truncation. This adds a per-iteration broadcast collective.", "status": "prototype", "title": "Enable Resource Governor", "type": "boolean" }, "enable_return_routed_experts": { "default": false, "description": "Router Replay (R3): capture per-token pre-EPLB logical top-k MoE expert ids so they can be returned on outputs (per request via SamplingParams.return_routed_experts), for train/inference routing alignment in MoE reinforcement learning. Zero overhead when disabled. Separated-routing MoE backends only.", "status": "beta", "title": "Enable Return Routed Experts", "type": "boolean" }, "enable_speculative_beam_history_d2h": { "default": false, "description": "Opt-in beam-search optimization: skip per-step beam-history D2H copies on likely-non-terminal steps via a host-side predictor and route the remaining copies through a private side stream. Mispredictions fall back to a synchronous .cpu(), preserving correctness but breaking overlap on that step. Incompatible with the async D2H worker (sampler_force_async_worker=True or confidential compute).", "status": "prototype", "title": "Enable Speculative Beam History D2H", "type": "boolean" }, "enable_tokenization_cache": { "default": false, "description": "Cache the tokenization of recent prompts so that a prompt extending a cached one only tokenizes its tail, which speeds up multi-turn serving with long prompts. Requires a fast tokenizer and applies only to prompts tokenized with add_special_tokens=False and no truncation. The output is identical to tokenizing the whole prompt.", "status": "prototype", "title": "Enable Tokenization Cache", "type": "boolean" }, "encode_only": { "default": false, "description": "Set to True to use the batch-forward encode() path, which runs a single forward pass and returns the model output directly, bypassing the scheduler and autoregressive loop. Works for encoder-only models (BERT, RoBERTa, reward models) and decoder models used in single-prefill mode (e.g., extracting embeddings). When False (default), uses the standard generate() path.", "status": "prototype", "title": "Encode Only", "type": "boolean" }, "encoder_cuda_graph_config": { "anyOf": [ { "$ref": "#/$defs/EncodeCudaGraphConfig" }, { "type": "null" } ], "default": null, "description": "CUDA graph configuration for the encoder forward pass of an encoder-decoder model. Use `cuda_graph_config` for the decoder and this field for the encoder. Encoder CUDA graphs require `encoder_max_batch_size` to be set.", "status": "prototype" }, "encoder_max_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of top-level encoder inputs processed in one iteration. For encoder-decoder models, each encoder request counts as one input. For multimodal models, each atomic image, video, or other encoder item counts as one input, even if it expands into multiple internal attention sequences. For encoder-decoder models, it also limits encoder CUDA graph batch sizes. Falls back to `max_batch_size` when unset.", "status": "prototype", "title": "Encoder Max Batch Size" }, "encoder_max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of encoder tokens. For encoder-decoder models, this limits encoder CUDA graph total-token buckets. For multimodal models, it limits encoder attention tokens scheduled in one iteration and is shared across all encoded modalities. It falls back to `max_num_tokens` when unset. Because an atomic multimodal item cannot be split, the effective budget is raised to the model's largest atomic item when necessary.", "status": "prototype", "title": "Encoder Max Num Tokens" }, "env_overrides": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won't update unless the code fetches them from os.environ on demand.", "status": "prototype", "title": "Env Overrides" }, "force_dynamic_quantization": { "default": false, "description": "If true, force dynamic quantization. Defaults to False.", "status": "prototype", "title": "Force Dynamic Quantization", "type": "boolean" }, "free_gpu_memory_fraction": { "anyOf": [ { "maximum": 1, "minimum": 0, "type": "number" }, { "type": "null" } ], "default": 0.9, "description": "The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.", "title": "Free Gpu Memory Fraction" }, "garbage_collection_gen0_threshold": { "default": 20000, "description": "Threshold for Python garbage collection of generation 0 objects. Lower values trigger more frequent garbage collection.", "status": "beta", "title": "Garbage Collection Gen0 Threshold", "type": "integer" }, "gather_generation_logits": { "default": false, "description": "Gather generation logits.", "status": "prototype", "title": "Gather Generation Logits", "type": "boolean" }, "generation_config": { "default": "trtllm", "description": "Controls whether sampling defaults are loaded from the model's generation_config.json. 'auto' applies supported values when the request does not specify them; 'trtllm' preserves TRT-LLM defaults. Precedence is request values, generation_config.json values, then TRT-LLM defaults.", "enum": [ "auto", "trtllm" ], "status": "prototype", "title": "Generation Config", "type": "string" }, "gms_config": { "$ref": "#/$defs/GmsConfig", "description": "GPU Memory Service (GMS) weight sharing config.", "status": "prototype" }, "gpus_per_node": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of GPUs per node.", "status": "beta", "title": "Gpus Per Node" }, "guided_decoding_backend": { "anyOf": [ { "enum": [ "xgrammar", "llguidance" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Guided decoding backend. llguidance is supported in PyTorch backend only.", "title": "Guided Decoding Backend" }, "hf_revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the model.", "title": "Revision" }, "internal_request_auth_key": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ] }, "iter_perf_stats_interval": { "default": 1, "description": "Build an iteration statistics record only every N executor iterations when enable_iter_perf_stats is true, which reduces the host overhead of collecting the statistics (including the per-request statistics of enable_iter_req_stats). A value of 1 builds a record every iteration. With N > 1, numCompletedRequests, numNewActiveRequests and the KV cache iteration deltas of skipped iterations are carried into a later record, so their totals stay exact but can be reported late. Without attention DP, one extra record is emitted when the last active request finishes in a skipped iteration; with attention DP, what is left after the last record of a busy period is reported when the executor resumes. All other fields describe only the sampled iteration, so Prometheus counters built from them (e.g. speculative decoding draft and accepted token totals) reach only about 1/N of the true totals, and gauges such as KV cache utilization refresh every N iterations. With enable_iter_req_stats, requests that finish in a skipped iteration get no requestStats entry.", "exclusiveMinimum": 0, "status": "prototype", "title": "Iter Perf Stats Interval", "type": "integer" }, "iter_stats_max_iterations": { "anyOf": [ { "minimum": -1, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of iterations for iter stats. Set to -1 to keep all iteration stats. Set to 0 to disable iteration stats in the TensorRT executor.", "status": "prototype", "title": "Iter Stats Max Iterations" }, "kv_cache_compression_config": { "anyOf": [ { "discriminator": { "mapping": { "quantization_for_cold_page": "#/$defs/ColdPageQuantizationCompressionConfig", "triattention": "#/$defs/TriAttentionKvCacheCompressionConfig" }, "propertyName": "algorithm" }, "oneOf": [ { "$ref": "#/$defs/ColdPageQuantizationCompressionConfig" }, { "$ref": "#/$defs/TriAttentionKvCacheCompressionConfig" } ] }, { "type": "null" } ], "default": null, "description": "KV-cache compression config; None disables compression.", "status": "prototype", "title": "Kv Cache Compression Config" }, "kv_cache_config": { "$ref": "#/$defs/KvCacheConfig", "description": "KV cache config." }, "kv_cache_dtype": { "default": "auto", "description": "The data type for the KV cache. 'auto' (default) leaves the checkpoint's own KV-cache quantization metadata untouched (quant_config.kv_cache_quant_algo is inherited as-is); 'fp8', 'fp8_ds_mla', or 'nvfp4' override it explicitly. 'fp8_ds_mla' selects the packed FP8 cache used by sparse MLA on SM90/SM120/SM121. Resolved at LLM-construction time, including when set via trtllm-serve --extra_llm_api_options.", "telemetry": { "allowed_values": [ "auto", "float16", "bfloat16", "float32", "fp8", "fp8_ds_mla", "nvfp4" ] }, "title": "Dtype", "type": "string" }, "kv_connector_config": { "anyOf": [ { "$ref": "#/$defs/KvCacheConnectorConfig" }, { "type": "null" } ], "default": null, "description": "The config for KV cache connector.", "status": "prototype" }, "layer_wise_benchmarks_config": { "$ref": "#/$defs/LayerwiseBenchmarksConfig", "status": "prototype" }, "load_format": { "anyOf": [ { "type": "string" }, { "$ref": "#/$defs/LoadFormat" } ], "default": 0, "description": "How to load the model weights. By default, detect the weight type from the model checkpoint.", "telemetry": { "allowed_values": [ "auto", "dummy", "vision_only", "gms" ] }, "title": "Load Format" }, "lora_config": { "anyOf": [ { "$ref": "#/$defs/LoraConfig" }, { "type": "null" } ], "default": null, "description": "LoRA configuration for the model." }, "max_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 2048, "description": "The maximum batch size.", "title": "Max Batch Size" }, "max_beam_width": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 1, "description": "The maximum beam width.", "title": "Max Beam Width" }, "max_input_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 1024, "description": "The maximum input length.", "title": "Max Input Len" }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 8192, "description": "The maximum number of tokens.", "title": "Max Num Tokens" }, "max_seq_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum sequence length.", "title": "Max Seq Len" }, "max_stats_len": { "default": 1000, "description": "The max number of performance statistic entries. Set to -1 to keep all entries. Set to 0 to use a minimum buffer size of 1.", "minimum": -1, "status": "prototype", "title": "Max Stats Len", "type": "integer" }, "mla_skip_correction_threshold": { "default": 8.0, "description": "Threshold for threshold-based skip-correction. This is used only when enable_mla_skip_correction is True. The default is 8. The maximum supported value depends on the selected kernel's BMM2 dtype: 8 for E4M3, 15 for FP16, and 32 for BF16.", "exclusiveMinimum": 0.0, "maximum": 32.0, "status": "prototype", "title": "Mla Skip Correction Threshold", "type": "number" }, "mm_encoder_only": { "default": false, "description": "Only load/execute the vision encoder part of the full model. Defaults to False.", "status": "prototype", "title": "Mm Encoder Only", "type": "boolean" }, "model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" } ], "description": "The path to the model checkpoint or the model name from the Hugging Face Hub.", "title": "Model" }, "model_kwargs": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "Optional parameters overriding model config defaults. Precedence: (1) model_kwargs, (2) model config file, (3) model config class defaults. Unknown keys are ignored", "status": "prototype", "title": "Model Kwargs" }, "modeling_v2": { "default": "off", "description": "Whether to build the model from a modeling_v2 target (experimental) instead of the built-in implementation. 'off' never does. 'auto' uses a target when one claims this deployment and falls back to the built-in implementation otherwise. 'require' raises instead of falling back.", "enum": [ "off", "auto", "require" ], "status": "prototype", "title": "Modeling V2", "type": "string" }, "moe_cluster_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The cluster parallel size for MoE model's expert weights.", "status": "deprecated", "title": "Moe Cluster Parallel Size" }, "moe_config": { "$ref": "#/$defs/MoeConfig", "description": "MoE config.", "status": "beta" }, "moe_expert_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The expert parallel size for MoE model's expert weights.", "title": "Moe Expert Parallel Size" }, "moe_tensor_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The tensor parallel size for MoE model's expert weights.", "title": "Moe Tensor Parallel Size" }, "multimodal_config": { "anyOf": [ { "$ref": "#/$defs/MultimodalConfig" }, { "type": "null" } ], "description": "Multimodal model configuration.", "status": "prototype" }, "mx_config": { "$ref": "#/$defs/ModelExpressConfig", "description": "ModelExpress (MX) P2P checkpoint loading config.", "status": "prototype" }, "num_instances": { "default": 1, "title": "Num Instances", "type": "integer" }, "num_postprocess_workers": { "default": 0, "description": "The number of processes used for postprocessing the generated tokens, including detokenization.", "status": "prototype", "title": "Num Postprocess Workers", "type": "integer" }, "num_serve_frontends": { "default": 1, "description": "The number of HTTP or OpenEngine frontend processes serving one executor. Used by trtllm-serve: values > 1 run additional attached frontend processes that share the serving port via SO_REUSEPORT (classic IPC executor path only).", "maximum": 64, "minimum": 1, "status": "prototype", "title": "Num Serve Frontends", "type": "integer" }, "nvfp4_gemm_config": { "$ref": "#/$defs/Nvfp4GemmConfig", "description": "NVFP4 GEMM backend config.", "status": "beta" }, "orchestrator_type": { "anyOf": [ { "enum": [ "rpc", "ray" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "The orchestrator type to use. Defaults to None, which uses MPI.", "status": "prototype", "title": "Orchestrator Type" }, "otlp_traces_endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Target URL to which OpenTelemetry traces will be sent.", "status": "prototype", "title": "Otlp Traces Endpoint" }, "peft_cache_config": { "anyOf": [ { "$ref": "#/$defs/PeftCacheConfig" }, { "type": "null" } ], "default": null, "description": "PEFT cache config.", "status": "prototype" }, "per_request_spec_decode_stats": { "default": false, "description": "Include per-request speculative-decoding acceptance statistics on each response choice. Server-side opt-in only: unlike return_perf_metrics this needs no per-request header, so benchmarking clients that discover the payload by shape do not have to know they are talking to TensorRT-LLM. Deliberately independent of return_perf_metrics, which also mounts the Prometheus endpoint. PyTorch backend only.", "status": "prototype", "title": "Per Request Spec Decode Stats", "type": "boolean" }, "perf_metrics_max_requests": { "default": 0, "description": "Deprecated compatibility field. Completed per-request metrics are no longer retained in memory.", "minimum": 0, "status": "deprecated", "title": "Perf Metrics Max Requests", "type": "integer" }, "perf_metrics_output_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Directory for per-process performance metrics JSONL files. Setting this enables collection even when return_perf_metrics is false.", "status": "prototype", "telemetry": { "exclude": true }, "title": "Perf Metrics Output Dir" }, "pipeline_parallel_size": { "default": 1, "description": "The pipeline parallel size.", "title": "Pipeline Parallel Size", "type": "integer" }, "post_processor_hook": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Python import path of a user post-processing hook applied after detokenization and before the per-endpoint response formatter (e.g. 'my_pkg.guardrail.MyPostProcessorHook'). The class must be importable and picklable, take no constructor arguments, and be callable as '__call__(chunk) -> verdict' (see tensorrt_llm.executor.postprocessor_hook). It runs once per output, per streaming chunk, and may rewrite, suppress, or terminate the output; it owns its own per-request state.", "status": "prototype", "title": "Post Processor Hook" }, "postprocess_tokenizer_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The path to the tokenizer directory for postprocessing.", "status": "prototype", "title": "Postprocess Tokenizer Dir" }, "pp_partition": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Pipeline parallel partition, a list of each rank's layer number.", "status": "prototype", "title": "Pp Partition" }, "prefill_capture_num_tokens": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Token-count buckets captured by the selected prefill CUDA graph implementation.", "status": "prototype", "title": "Prefill Capture Num Tokens" }, "prefill_cuda_graph_backend": { "$ref": "#/$defs/PrefillCudaGraphBackend", "default": "disabled", "description": "CUDA graph implementation used for prefill requests. Defaults to disabled.", "status": "prototype" }, "print_iter_log": { "default": false, "description": "Print iteration logs.", "status": "beta", "title": "Print Iter Log", "type": "boolean" }, "prometheus_metrics_config": { "anyOf": [ { "$ref": "#/$defs/PrometheusMetricsConfig" }, { "type": "null" } ], "default": null, "description": "Configuration for Prometheus metrics collection, including custom histogram bucket boundaries.", "status": "prototype" }, "ray_placement_config": { "anyOf": [ { "$ref": "#/$defs/RayPlacementConfig" }, { "type": "null" } ], "default": null, "description": "Placement config for RayGPUWorker. Only used with AsyncLLM and orchestrator_type='ray'.", "status": "prototype" }, "ray_worker_extension_cls": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The full worker extension class name including module path. Allows users to extend the functions of the RayGPUWorker class.", "status": "prototype", "title": "Ray Worker Extension Cls" }, "ray_worker_nsight_options": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Nsight options.", "status": "prototype", "title": "Ray Worker Nsight Options" }, "reasoning_parser": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The parser to separate reasoning content from output.", "status": "prototype", "telemetry": { "allowed_values": [ "auto", "deepseek-r1", "poolside_v1", "laguna", "qwen3", "qwen3_5", "minimax_m2", "minimax_m2_append_think", "nano-v3", "gemma4", "kimi_k2", "kimi_k25", "k-exaone" ] }, "title": "Reasoning Parser" }, "reorder_policy_config": { "anyOf": [ { "$ref": "#/$defs/ReorderRequestPolicyConfig" }, { "type": "null" } ], "default": null, "description": "The request reordering policy to use.", "status": "prototype" }, "request_stats_max_iterations": { "anyOf": [ { "minimum": -1, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of iterations for request stats. Set to -1 to keep all request stats. Set to 0 to disable request stats.", "status": "prototype", "title": "Request Stats Max Iterations" }, "return_perf_metrics": { "default": false, "description": "Allow serving responses to include per-request performance metrics when the request sets X-TRTLLM-return-metrics: 1.", "status": "prototype", "title": "Return Perf Metrics", "type": "boolean" }, "revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the model.", "title": "Revision" }, "router": { "additionalProperties": true, "properties": { "type": { "default": "round_robin", "title": "Type", "type": "string" } }, "type": "object" }, "sampler_force_async_worker": { "default": false, "description": "Force usage of the async worker in the sampler for D2H copies, even if confidential compute is not active. Normally, the async worker should only be used when confidential compute is active. This argument is provided to enable it for testing purposes, irrespective of confidential compute state.", "status": "prototype", "title": "Sampler Force Async Worker", "type": "boolean" }, "scheduler_config": { "$ref": "#/$defs/SchedulerConfig", "description": "Scheduler config.", "status": "prototype" }, "skip_tokenizer_init": { "default": false, "description": "Whether to skip the tokenizer initialization.", "title": "Skip Tokenizer Init", "type": "boolean" }, "sleep_config": { "anyOf": [ { "$ref": "#/$defs/SleepConfig" }, { "type": "null" } ], "default": null, "description": "Configuration for the LLM sleep feature. Sleep feature requires extra setup that may slow down model loading. Only enable it if you intend to use this feature.", "status": "prototype" }, "sparse_attention_config": { "anyOf": [ { "discriminator": { "mapping": { "csa2": "#/$defs/CSA2SparseAttentionConfig", "deepseek_v4": "#/$defs/DeepSeekV4SparseAttentionConfig", "dsa": "#/$defs/DeepSeekSparseAttentionConfig", "minimax_m3": "#/$defs/MiniMaxM3SparseAttentionConfig", "qsa": "#/$defs/QSASparseAttentionConfig", "rocket": "#/$defs/RocketSparseAttentionConfig", "skip_softmax": "#/$defs/SkipSoftmaxAttentionConfig" }, "propertyName": "algorithm" }, "oneOf": [ { "$ref": "#/$defs/QSASparseAttentionConfig" }, { "$ref": "#/$defs/RocketSparseAttentionConfig" }, { "$ref": "#/$defs/DeepSeekSparseAttentionConfig" }, { "$ref": "#/$defs/DeepSeekV4SparseAttentionConfig" }, { "$ref": "#/$defs/CSA2SparseAttentionConfig" }, { "$ref": "#/$defs/SkipSoftmaxAttentionConfig" }, { "$ref": "#/$defs/MiniMaxM3SparseAttentionConfig" } ] }, { "type": "null" } ], "default": null, "description": "Sparse attention config.", "status": "prototype", "title": "Sparse Attention Config" }, "speculative_config": { "anyOf": [ { "discriminator": { "mapping": { "AUTO": "#/$defs/AutoDecodingConfig", "DFlash": "#/$defs/DFlashDecodingConfig", "DSpark": "#/$defs/DSparkDecodingConfig", "Draft_Target": "#/$defs/DraftTargetDecodingConfig", "Eagle": "#/$defs/EagleDecodingConfig", "Eagle3": "#/$defs/Eagle3DecodingConfig", "MTP": "#/$defs/MTPDecodingConfig", "NGram": "#/$defs/NGramDecodingConfig", "PARD": "#/$defs/PARDDecodingConfig", "SA": "#/$defs/SADecodingConfig", "SaveState": "#/$defs/SaveHiddenStatesDecodingConfig", "User_Provided": "#/$defs/UserProvidedDecodingConfig" }, "propertyName": "decoding_type" }, "oneOf": [ { "$ref": "#/$defs/DraftTargetDecodingConfig" }, { "$ref": "#/$defs/Eagle3DecodingConfig" }, { "$ref": "#/$defs/EagleDecodingConfig" }, { "$ref": "#/$defs/MTPDecodingConfig" }, { "$ref": "#/$defs/NGramDecodingConfig" }, { "$ref": "#/$defs/SADecodingConfig" }, { "$ref": "#/$defs/UserProvidedDecodingConfig" }, { "$ref": "#/$defs/SaveHiddenStatesDecodingConfig" }, { "$ref": "#/$defs/PARDDecodingConfig" }, { "$ref": "#/$defs/DFlashDecodingConfig" }, { "$ref": "#/$defs/DSparkDecodingConfig" }, { "$ref": "#/$defs/AutoDecodingConfig" } ] }, { "type": "null" } ], "default": null, "description": "Speculative decoding config.", "title": "Speculative Config" }, "stream_interval": { "default": 1, "description": "The iteration interval to create responses under the streaming mode. Set this to a larger value when the batch size is large, which helps reduce the streaming overhead.", "exclusiveMinimum": 0, "title": "Stream Interval", "type": "integer" }, "telemetry_config": { "anyOf": [ { "$ref": "#/$defs/TelemetryConfig", "description": "Telemetry configuration (opt-out, usage context).", "status": "prototype" }, { "type": "null" } ] }, "tensor_parallel_size": { "default": 1, "description": "The tensor parallel size.", "title": "Tensor Parallel Size", "type": "integer" }, "tokenizer": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "not": {} }, { "type": "null" } ], "default": null, "description": "The path to the tokenizer checkpoint or the tokenizer name from the Hugging Face Hub.", "title": "Tokenizer" }, "tokenizer_mode": { "default": "auto", "description": "The mode to initialize the tokenizer.", "enum": [ "auto", "slow" ], "title": "Tokenizer Mode", "type": "string" }, "tokenizer_revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the tokenizer.", "title": "Tokenizer Revision" }, "torch_compile_config": { "anyOf": [ { "$ref": "#/$defs/TorchCompileConfig" }, { "type": "null" } ], "default": null, "description": "Torch compile config.", "status": "prototype" }, "trust_remote_code": { "default": false, "description": "Whether to trust the remote code.", "title": "Trust Remote Code", "type": "boolean" }, "urls": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "title": "Urls" }, "use_cute_dsl_bf16_bmm": { "default": false, "description": "If true, use CuTe DSL BF16 BMM on Blackwell (SM100/SM103) and Rubin (SM107), including the DeepSeek-V4 o_a projection on Rubin when DSL support and dimension alignment permit. Defaults to false; automatically enabled with pipeline_parallel_size > 1 on SM100/SM103/SM107.", "status": "prototype", "title": "Use Cute Dsl Bf16 Bmm", "type": "boolean" }, "use_cute_dsl_bf16_gemm": { "default": false, "description": "If true, use CuTe DSL BF16 persistent GEMM for Linear layers on Blackwell (SM100/SM103) and Rubin (SM107) when supported. Defaults to false; automatically enabled with pipeline_parallel_size > 1 on SM100/SM103/SM107.", "status": "prototype", "title": "Use Cute Dsl Bf16 Gemm", "type": "boolean" }, "use_cute_dsl_blockscaling_bmm": { "default": false, "description": "If true, use CuTe DSL fp8 blockscaling bmm implementation.", "status": "prototype", "title": "Use Cute Dsl Blockscaling Bmm", "type": "boolean" }, "use_cute_dsl_blockscaling_mm": { "default": false, "description": "If true, use CuTe DSL fp8 blockscaling mm implementation.", "status": "prototype", "title": "Use Cute Dsl Blockscaling Mm", "type": "boolean" }, "use_fine_grained_sync": { "default": false, "description": "Enable fine-grained synchronization for MoE kernels on SM107. The FC1 producer kernel signals per-tile completion flags in device memory and the FC2 consumer kernel waits on them, so the two GEMMs overlap instead of serializing at kernel launch boundaries.", "status": "prototype", "title": "Use Fine Grained Sync", "type": "boolean" }, "video_pruning_rate": { "anyOf": [ { "exclusiveMaximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Pruning rate for video frames in multimodal models for Efficient Video Sampling (EVS). NOTE: this is currently only implemented in nemotron multimodal models. None (default) disables EVS, values in [0, 1) enable pruning.", "status": "prototype", "title": "Video Pruning Rate" } }, "type": "object" }, "DraftTargetDecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "Draft_Target", "default": "Draft_Target", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "DraftTargetDecodingConfig", "type": "object" }, "DwdpConfig": { "additionalProperties": false, "description": "Configuration for Distributed Weight Data Parallelism (DWDP).\n\nDWDP accelerates the context (prefill) phase of disaggregated MoE serving\nby combining data parallelism with NVLink-based expert weight sharing.\nEach worker holds a subset of experts locally and asynchronously prefetches\nthe remaining experts from peer workers via CUDA VMM + MNNVL fabric\nhandles (composite virtual-address layout with zero-copy local mapping and\ndouble-buffered P2P remote regions), enabling fully asynchronous execution\nacross ranks without synchronization barriers.\n\nCurrently supported with the CuteDSL MoE backend and NVFP4 quantization\non MNNVL-connected multi-GPU systems (e.g., GB200).", "properties": { "contention_opt": { "default": false, "description": "Enable limited-contention prefetch optimization. Uses batched memcpy with round-robin slice ordering across peers and a 2 MiB slice granularity to reduce NVLink contention.", "title": "Contention Opt", "type": "boolean" }, "dwdp_size": { "default": 1, "description": "The number of GPUs per DWDP group.", "title": "Dwdp Size", "type": "integer" }, "num_experts_per_worker": { "default": 0, "description": "The number of experts per worker.", "title": "Num Experts Per Worker", "type": "integer" }, "num_groups": { "default": 1, "description": "The number of DWDP groups. Total workers = num_groups * dwdp_size.", "title": "Num Groups", "type": "integer" }, "num_prefetch_experts": { "default": 0, "description": "The number of prefetch experts per worker.", "title": "Num Prefetch Experts", "type": "integer" } }, "title": "DwdpConfig", "type": "object" }, "DynamicBatchConfig": { "additionalProperties": false, "description": "Dynamic batch configuration.\n\nControls how batch size and token limits are dynamically adjusted at runtime.", "properties": { "dynamic_batch_moving_average_window": { "default": 128, "description": "The window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens", "title": "Dynamic Batch Moving Average Window", "type": "integer" }, "enable_batch_size_tuning": { "default": true, "description": "Controls if the batch size should be tuned dynamically", "title": "Enable Batch Size Tuning", "type": "boolean" }, "enable_max_num_tokens_tuning": { "default": false, "description": "Controls if the max num tokens should be tuned dynamically", "title": "Enable Max Num Tokens Tuning", "type": "boolean" } }, "title": "DynamicBatchConfig", "type": "object" }, "Eagle3DecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "Eagle3", "default": "Eagle3", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "dynamic_tree_max_topK": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The topK value for each layer when dynamic tree is enabled. Required when use_dynamic_tree is True; ignored (with a warning) when use_dynamic_tree is False.", "title": "Dynamic Tree Max Topk" }, "eagle3_layers_to_capture": { "anyOf": [ { "items": { "type": "integer" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "default": null, "description": "Target model layer indices to capture hidden states from for the EAGLE3 draft model. Defaults to {1, num_layers//2-1, num_layers-4}.", "title": "Eagle3 Layers To Capture" }, "eagle3_model_arch": { "default": "llama3", "description": "The model architecture of the eagle3 model.", "enum": [ "llama3", "mistral_large3" ], "title": "Eagle3 Model Arch", "type": "string" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_non_leaves_per_layer": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of non-leaves in each layer.", "title": "Max Non Leaves Per Layer" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "num_eagle_layers": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Deprecated TensorRT-only field with different semantics from draft model layer count. Do not use on the PyTorch backend.", "title": "Num Eagle Layers" }, "sa_config": { "anyOf": [ { "$ref": "#/$defs/SAEnhancerConfig" }, { "type": "null" } ], "default": null, "description": "Optional Suffix Automaton configuration. When set, combines SA drafting with Eagle3 speculative decoding.", "status": "beta" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_dynamic_tree": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": false, "description": "Whether to use dynamic tree (Eagle-2 algorithm).", "title": "Use Dynamic Tree" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "Eagle3DecodingConfig", "type": "object" }, "EagleDecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "Eagle", "default": "Eagle", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "dynamic_tree_max_topK": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The topK value for each layer when dynamic tree is enabled. Required when use_dynamic_tree is True; ignored (with a warning) when use_dynamic_tree is False.", "title": "Dynamic Tree Max Topk" }, "eagle3_layers_to_capture": { "anyOf": [ { "items": { "type": "integer" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "default": null, "description": "Target model layer indices to capture hidden states from for the EAGLE3 draft model. Defaults to {1, num_layers//2-1, num_layers-4}.", "title": "Eagle3 Layers To Capture" }, "eagle3_model_arch": { "default": "llama3", "description": "The model architecture of the eagle3 model.", "enum": [ "llama3", "mistral_large3" ], "title": "Eagle3 Model Arch", "type": "string" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_non_leaves_per_layer": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of non-leaves in each layer.", "title": "Max Non Leaves Per Layer" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "num_eagle_layers": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Deprecated TensorRT-only field with different semantics from draft model layer count. Do not use on the PyTorch backend.", "title": "Num Eagle Layers" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_dynamic_tree": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": false, "description": "Whether to use dynamic tree (Eagle-2 algorithm).", "title": "Use Dynamic Tree" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "EagleDecodingConfig", "type": "object" }, "EncodeCudaGraphConfig": { "additionalProperties": false, "description": "CUDA graph configuration for encode-only requests (``LLM.encode()``).\n\nA graph is captured at startup for every feasible combination of\n``batch_sizes``, ``num_tokens`` and ``seq_lens``. A batch runs the graph\nfor its (batch size, total tokens, longest request) shape, rounded up when\n``enable_padding`` is set, and runs eagerly if no graph matches.\n\nExample::\n\n cuda_graph_config = EncodeCudaGraphConfig(\n batch_sizes=[1, 2, 4, 8],\n num_tokens=[128, 256, 512],\n seq_lens=[64, 128],\n enable_padding=True,\n extra_model_inputs=[\n EncodeExtraInputSpec(name=\"token_type_ids\",\n shape=(\"num_tokens\",),\n dtype=\"int32\"),\n ],\n )\n llm = LLM(model, encode_only=True, cuda_graph_config=cuda_graph_config)\n # token_type_ids: one value per token, prompts packed in order.\n outputs = llm.encode(prompts, token_type_ids=token_type_ids)", "properties": { "batch_sizes": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of batch sizes to create CUDA graphs for.", "title": "Batch Sizes" }, "enable_padding": { "default": false, "description": "If true, batches are rounded up to the nearest cuda_graph_batch_size. This is usually a net win for performance.", "title": "Enable Padding", "type": "boolean" }, "extra_model_inputs": { "description": "Tensor kwargs (beyond input_ids / position_ids) that LLM.encode() passes to the encoder forward() under CUDA graphs. Each is backed by a static buffer sized along its symbolic dim at the bucket maximum (`num_tokens` \u2192 `max(num_tokens)`, `batch_size` \u2192 `max(batch_sizes)`). With encoder CUDA graphs enabled, every tensor kwarg passed to encode() must be declared here, and every declared input must be passed on every call. Tensors may be on host or device.", "items": { "$ref": "#/$defs/EncodeExtraInputSpec" }, "title": "Extra Model Inputs", "type": "array" }, "max_batch_size": { "default": 0, "description": "Maximum batch size for CUDA graphs.", "minimum": 0, "title": "Max Batch Size", "type": "integer" }, "max_num_token": { "default": 0, "description": "Maximum total number of tokens for encoder CUDA graphs. If `num_tokens` is provided, must equal max(num_tokens); otherwise `num_tokens` is generated from this value. Ignored by a fixed-shape feature encoder.", "minimum": 0, "title": "Max Num Token", "type": "integer" }, "max_seq_len": { "default": 0, "description": "Maximum per-request sequence length for encoder CUDA graphs. If `seq_lens` is provided, must equal max(seq_lens); otherwise `seq_lens` is generated from this value. Ignored by a fixed-shape feature encoder.", "minimum": 0, "title": "Max Seq Len", "type": "integer" }, "mode": { "const": "encode", "default": "encode", "description": "CUDA graph configuration mode.", "title": "Mode", "type": "string" }, "num_tokens": { "anyOf": [ { "items": { "exclusiveMinimum": 0, "type": "integer" }, "minItems": 1, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of total token counts (sum of all per-request sequence lengths in a batch) to create encoder CUDA graphs for. Required for an encoder that packs a variable number of tokens per request; ignored by an encoder whose input is a fixed-shape per-request feature tensor, which derives this from the model.", "title": "Num Tokens" }, "seq_lens": { "anyOf": [ { "items": { "exclusiveMinimum": 0, "type": "integer" }, "minItems": 1, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of max per-request sequence lengths to create encoder CUDA graphs for. Required for an encoder that packs a variable number of tokens per request; ignored by an encoder whose input is a fixed-shape per-request feature tensor, which derives this from the model.", "title": "Seq Lens" } }, "title": "EncodeCudaGraphConfig", "type": "object" }, "EncodeExtraInputSpec": { "additionalProperties": false, "description": "Declares an extra encoder-forward tensor kwarg for CUDA graph capture.\n\nOne spec is required for every tensor kwarg passed to\n`LLM.encode(..., **model_kwargs)` (e.g. `token_type_ids` for BERT). The\nrunner backs each with a static buffer sized at the bucket maximum, and\nwarmup captures every bucket with a zero-filled stand-in so the graph sees\nthe full forward signature before the first real call.\n\nOnly tensors can be declared: a non-tensor kwarg cannot be captured and\ninstead forces that `encode()` call onto the eager path. Device is not part\nof the spec: pass tensors where they already live. Each replay copies them\ninto the graph's buffer asynchronously, H2D for host tensors and D2D for\ndevice tensors.", "properties": { "dtype": { "description": "Tensor dtype string accepted by tensorrt_llm (e.g. \"int32\", \"float32\", \"bfloat16\"). See `tensorrt_llm._utils._str_to_torch_dtype_dict` for the full list.", "title": "Dtype", "type": "string" }, "name": { "description": "Kwarg name as it appears in the encoder forward() signature (e.g. \"token_type_ids\").", "title": "Name", "type": "string" }, "shape": { "description": "Tensor shape. Use exactly one symbolic dim from {\"num_tokens\", \"batch_size\"}: \"num_tokens\" scales with the packed token bucket (e.g. token_type_ids), \"batch_size\" scales with the request bucket (e.g. per-request features that are independent of sequence length). Remaining dims must be positive integer literals (e.g. hidden_size).", "items": { "anyOf": [ { "enum": [ "num_tokens", "batch_size" ], "type": "string" }, { "exclusiveMinimum": 0, "type": "integer" } ] }, "title": "Shape", "type": "array" } }, "required": [ "name", "shape", "dtype" ], "title": "EncodeExtraInputSpec", "type": "object" }, "GmsConfig": { "additionalProperties": false, "description": "Prototype configuration for GPU Memory Service (GMS) weight sharing.\n\nComposes orthogonally with ``checkpoint_format`` on ``TorchLlmArgs``:\nGMS controls *where* weights live (a node-shared GPU memory pool\nso multiple TRT-LLM instances zero-copy share the same bytes),\nwhile ``checkpoint_format`` controls *how* they get there (HF disk,\nMX P2P, etc.). Effective only when ``TorchLlmArgs.load_format ==\nLoadFormat.GMS``; setting any non-default field here without that\nload_format triggers a warning via :meth:`TorchLlmArgs.validate_gms_config`.\n\nTwo roles are decided at connect time by the GMS daemon, not by\nconfig:\n\n- **RW (writer)**: the first instance loads weights into the pool\n via the normal checkpoint-loader pipeline, then commits them.\n- **RO (reader)**: subsequent instances zero-copy materialize the\n already-committed layout \u2014 no disk I/O, no per-instance copies.\n\nSee :class:`tensorrt_llm._torch.memory.gpu_memory_backend.GMSBackend`\nfor the integration adapter and the ``LoadFormat.GMS`` branch in\n``ModelLoader.load`` for orchestration.", "properties": { "mode": { "default": "auto", "description": "GMS operating mode: 'auto' requests RW or RO, 'rw' requires writer mode, and 'ro' requires read-only mode.", "enum": [ "auto", "rw", "ro" ], "status": "prototype", "title": "Mode", "type": "string" }, "socket_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Unix domain socket path of the GPU Memory Service. When unset, the GMS library resolves its default per-device path.", "status": "prototype", "title": "Socket Path" }, "tag": { "default": "weights", "description": "Tag identifying the model weight set in the GMS memory pool. Defaults to 'weights' to match the GMS library convention. The tag must uniquely identify a compatible model/parallelism layout for a given GMS daemon; reusing a tag for different weights can make RO readers attach to the wrong pool.", "status": "prototype", "title": "Tag", "type": "string" } }, "title": "GmsConfig", "type": "object" }, "KVEventsConfig": { "additionalProperties": false, "description": "Configuration for streaming (push-based) KV cache event publishing.", "properties": { "buffer_steps": { "default": 10000, "description": "Number of previously published batches retained for replay.", "exclusiveMinimum": 0, "title": "Buffer Steps", "type": "integer" }, "enable_kv_cache_events": { "default": false, "description": "Whether to produce and publish KV cache events over the streaming (push) path.", "title": "Enable Kv Cache Events", "type": "boolean" }, "endpoint": { "default": "tcp://*:5557", "description": "Base ZeroMQ endpoint the publisher binds. Each attention-DP rank binds base_port+rank, so co-located engines (e.g. disaggregated prefill and decode on one host) must use distinct base ports.", "minLength": 1, "title": "Endpoint", "type": "string" }, "hwm": { "default": 100000, "description": "ZeroMQ publisher socket high-water mark. 0 means unlimited in ZeroMQ, so it is disallowed here.", "exclusiveMinimum": 0, "title": "Hwm", "type": "integer" }, "max_queue_size": { "default": 100000, "description": "Maximum number of batches queued for background publishing. Must be positive; 0 would make the queue unbounded.", "exclusiveMinimum": 0, "title": "Max Queue Size", "type": "integer" }, "publisher": { "anyOf": [ { "enum": [ "null", "zmq" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Publisher implementation. Defaults to 'zmq' when events are enabled and 'null' otherwise.", "title": "Publisher" }, "replay_endpoint": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ], "default": null, "description": "Optional base ZeroMQ endpoint used to replay KV cache events. Ranks apply the same global base_port+rank convention as `endpoint`. Only ranks sharing a host contend for a port, so the two base ports must be at least ranks-per-host apart or a rank's replay bind collides with another rank's publish bind on that host.", "title": "Replay Endpoint" }, "topic": { "default": "", "description": "ZeroMQ subscription topic used for KV cache event batches.", "title": "Topic", "type": "string" } }, "title": "KVEventsConfig", "type": "object" }, "KvCacheConfig": { "additionalProperties": false, "description": "Configuration for the KV cache.", "properties": { "attention_dp_events_gather_period_ms": { "default": 5, "description": "The period in milliseconds to gather attention DP events across ranks.", "title": "Attention Dp Events Gather Period Ms", "type": "integer" }, "avg_seq_len": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "Average total sequence length of the serving workload, used to build the KV cache manager v2 typical step for hybrid Mamba models and DeepSeek-V4. Hybrid Mamba models warn and fall back to half of max_seq_len when this is unset. This does not take effect when pool_ratio is set.", "status": "prototype", "title": "Avg Seq Len" }, "block_reuse_config": { "$ref": "#/$defs/BlockReuseConfig", "description": "KV cache manager v2 configuration for block reuse policies.", "status": "prototype" }, "copy_on_partial_reuse": { "default": true, "description": "Whether partially matched blocks that are in use can be reused after copying them.", "title": "Copy On Partial Reuse", "type": "boolean" }, "cross_kv_cache_fraction": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "description": "The fraction of the KV Cache memory should be reserved for cross attention. If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Defaults to None (unset); must be set when using an encoder-decoder model and must not be set otherwise.", "title": "Cross Kv Cache Fraction" }, "disk_cache_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Directory used for disk KV cache files. Must be set when `disk_cache_size` is positive.", "title": "Disk Cache Path" }, "disk_cache_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "Size of the disk cache in bytes. Only used by KV cache manager v2 in the PyTorch backend.", "title": "Disk Cache Size" }, "disk_prefetch_num_reqs": { "default": 0, "description": "Number of queued context requests to prefetch disk-tier KV cache blocks to host for. Set to 0 to disable prefetch. Only effective with KV cache manager v2 and block reuse enabled.", "minimum": 0, "title": "Disk Prefetch Num Reqs", "type": "integer" }, "dtype": { "default": "auto", "description": "The data type for the KV cache. 'auto' (default) leaves the checkpoint's own KV-cache quantization metadata untouched (quant_config.kv_cache_quant_algo is inherited as-is); 'fp8', 'fp8_ds_mla', or 'nvfp4' override it explicitly. 'fp8_ds_mla' selects the packed FP8 cache used by sparse MLA on SM90/SM120/SM121. Resolved at LLM-construction time, including when set via trtllm-serve --extra_llm_api_options.", "telemetry": { "allowed_values": [ "auto", "float16", "bfloat16", "float32", "fp8", "fp8_ds_mla", "nvfp4" ] }, "title": "Dtype", "type": "string" }, "enable_block_reuse": { "default": true, "description": "Controls if KV cache blocks can be reused for different requests.", "title": "Enable Block Reuse", "type": "boolean" }, "enable_kv_pool_rebalance": { "default": false, "description": "Opt in to the KVCacheManagerV2 auto-tuner (``adjust()``) for rebalancing pool-group ratios between iterations. When True the PyExecutor calls ``adjust()`` opportunistically; the auto-tuner itself remains gated by the ``kv_pool_rebalance_config`` thresholds (2000 samples / 120s by default). When False (default) the rebalance hook is skipped entirely and pool ratios remain at their warmup-derived values. Beta: enable at your own risk. Only used when using KV cache manager v2 (experimental). This option is incompatible with dtype='fp8_ds_mla'.", "status": "prototype", "title": "Enable Kv Pool Rebalance", "type": "boolean" }, "enable_partial_reuse": { "default": true, "description": "Whether blocks that are only partially matched can be reused.", "title": "Enable Partial Reuse", "type": "boolean" }, "enable_swa_scratch_reuse": { "default": false, "description": "Whether KV cache manager v2 uses SWA scratch reuse during prefill.", "status": "prototype", "title": "Enable Swa Scratch Reuse", "type": "boolean" }, "event_buffer_max_size": { "default": 0, "description": "Maximum size of the event buffer. If set to 0, the event buffer will not be used.", "title": "Event Buffer Max Size", "type": "integer" }, "fp8_context_mla_kv_len_cap": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Override, in tokens, for the max summed attended-KV length (total_kv_len) per forward step that the fp8 context-MLA attention workspace is reserved and scheduled for. Only affects fp8 context-MLA models (e.g. DeepSeek / Kimi with an fp8 KV cache). None (default) reserves for the never-stall worst case min(max_batch_size, max_num_tokens) * max_seq_len. A smaller value reserves less workspace (freeing KV cache) and defers context requests whose summed attended KV would exceed it; it is floored at max_seq_len and capped at the worst case. Safe at any value -- the scheduler enforces it -- trading prefill batching under heavy reuse for KV cache capacity.", "status": "prototype", "title": "Fp8 Context Mla Kv Len Cap" }, "free_gpu_memory_fraction": { "anyOf": [ { "maximum": 1, "minimum": 0, "type": "number" }, { "type": "null" } ], "default": 0.9, "description": "The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.", "title": "Free Gpu Memory Fraction" }, "host_cache_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Size of the host cache in bytes. If both `max_tokens` and `host_cache_size` are specified, memory corresponding to the minimum will be used.", "title": "Host Cache Size" }, "iteration_stats_interval": { "default": 1, "description": "How often (in iterations) to collect per-iteration KV cache statistics. A value of 1 means every iteration; a value of N means every Nth iteration. Between collections, the C++ deltas accumulate, so the reported deltas cover N iterations.", "exclusiveMinimum": 0, "title": "Iteration Stats Interval", "type": "integer" }, "kv_cache_event_hash_algo": { "default": "auto", "description": "The block hash algorithm used by KV cache manager events. 'auto' uses the native hash for each KV cache manager. Explicit V2 hash choices are ignored with a warning by the V1 KV cache manager.", "enum": [ "auto", "v1_block_key", "v2_sha256", "v2_sha256_64" ], "status": "prototype", "title": "Kv Cache Event Hash Algo", "type": "string" }, "kv_events_config": { "anyOf": [ { "$ref": "#/$defs/KVEventsConfig" }, { "type": "null" } ], "default": null, "description": "Streaming (push-based) KV cache event publishing (KV cache manager V2 only). When set, each rank publishes its own events directly (e.g. over ZeroMQ) instead of the buffered event_buffer_max_size gather/poll path.", "status": "prototype" }, "kv_pool_rebalance_config": { "$ref": "#/$defs/KvPoolRebalanceConfig", "description": "Tuning for the KVCacheManagerV2 pool rebalance auto-tuner. Only used when ``enable_kv_pool_rebalance`` is True.", "status": "prototype" }, "mamba_ssm_cache_dtype": { "default": "auto", "description": "The data type to use for the Mamba SSM cache. If set to 'auto', the data type will be inferred from the model config.", "enum": [ "auto", "float16", "bfloat16", "float32" ], "title": "Mamba Ssm Cache Dtype", "type": "string" }, "mamba_ssm_philox_rounds": { "default": 10, "description": "Number of Philox rounds for stochastic rounding PRNG. Higher values give better randomness but increase compute cost. Only used when mamba_ssm_stochastic_rounding is enabled.", "minimum": 1, "title": "Mamba Ssm Philox Rounds", "type": "integer" }, "mamba_ssm_stochastic_rounding": { "default": false, "description": "Enable stochastic rounding for Mamba SSM state updates. Only applicable with float16 cache dtype.", "title": "Mamba Ssm Stochastic Rounding", "type": "boolean" }, "mamba_state_cache_interval": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "Deprecated alias for mamba_state_config.periodic_snapshot_interval.", "status": "deprecated", "telemetry": { "exclude": true }, "title": "Mamba State Cache Interval" }, "mamba_state_config": { "$ref": "#/$defs/MambaStateConfig", "description": "Configuration for reusable Mamba state snapshots." }, "max_attention_window": { "anyOf": [ { "items": { "type": "integer" }, "minItems": 1, "type": "array" }, { "type": "null" } ], "default": null, "description": "Size of the attention window for each sequence. Only the last tokens will be stored in the KV cache. If the number of elements in `max_attention_window` is less than the number of layers, `max_attention_window` will be repeated multiple times to the number of layers.", "title": "Max Attention Window" }, "max_gpu_total_bytes": { "default": 0, "description": "The maximum size in bytes of GPU memory that can be allocated for the KV cache. If both `max_gpu_total_bytes` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be allocated.", "minimum": 0, "title": "Max Gpu Total Bytes", "type": "integer" }, "max_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of tokens that should be stored in the KV cache. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.", "title": "Max Tokens" }, "max_util_for_resume": { "default": 0.95, "description": "The maximum utilization of the KV cache for resume. Default is 95%. Only used when using KV cache manager v2 (experimental).", "exclusiveMinimum": 0, "maximum": 1, "status": "prototype", "title": "Max Util For Resume", "type": "number" }, "pool_ratio": { "anyOf": [ { "items": { "type": "number" }, "minItems": 1, "type": "array" }, { "type": "null" } ], "default": null, "description": "Initial hot-tier byte ratios by layer group for KV cache manager v2. Values map to KVCacheManagerV2 layer-group ID order and must sum to 1.0. Cold tiers preserve the implied slot-count ratios. Hybrid Mamba models and DeepSeek-V4 use this directly, so avg_seq_len does not take effect when this is set.", "status": "prototype", "title": "Pool Ratio" }, "secondary_offload_min_priority": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Only blocks with priority > secondary_offload_min_priority can be offloaded to secondary memory.", "title": "Secondary Offload Min Priority" }, "sink_token_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "deprecated": true, "description": "Deprecated and ignored on the PyTorch backend. StreamingLLM is not supported by the PyTorch attention kernels \u2014 any non-None value has no effect and will be silently dropped before reaching the executor.", "title": "Sink Token Length" }, "tokens_per_block": { "default": 32, "description": "The number of tokens per block.", "title": "Tokens Per Block", "type": "integer" }, "use_kv_cache_manager_v2": { "anyOf": [ { "type": "boolean" }, { "const": "auto", "type": "string" } ], "default": true, "description": "Whether to use the KV cache manager v2 (experimental). Defaults to True. Set False to select V1 for models that support it. 'auto' uses the model-specific preference and falls back to False when the model does not declare one.", "status": "prototype", "title": "Use Kv Cache Manager V2" }, "use_uvm": { "default": false, "description": "Whether to use UVM for the KV cache.", "title": "Use Uvm", "type": "boolean" } }, "title": "KvCacheConfig", "type": "object" }, "KvCacheConnectorConfig": { "additionalProperties": false, "description": "Configuration for the KV Cache Connector.\n\nCan be configured either by specifying a named preset via ``connector``\n(e.g. ``\"lmcache\"``), or by providing explicit ``connector_module``,\n``connector_scheduler_class``, and ``connector_worker_class`` fields.\nWhen ``connector`` is set, the module/class fields are auto-populated\nfrom the preset registry and can be omitted.", "properties": { "connector": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Named connector preset (e.g. 'lmcache'). When set, connector_module/scheduler_class/worker_class are auto-populated from the preset registry.", "telemetry": { "allowed_values": [ "lmcache", "lmcache-mp", "kvbm", "mooncake-store" ] }, "title": "Connector" }, "connector_module": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The import path to the connector module. It will be imported with `importlib.import_module`.", "title": "Connector Module" }, "connector_scheduler_class": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The class name of the scheduler within the module.", "title": "Connector Scheduler Class" }, "connector_worker_class": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The class name of the worker within the module.", "title": "Connector Worker Class" }, "mooncake_store": { "anyOf": [ { "$ref": "#/$defs/MooncakeStoreConfig" }, { "type": "null" } ], "default": null, "description": "Pool topology for the 'mooncake-store' connector. When set, trtllm-serve provisions the pool during bringup instead of requiring MOONCAKE_CONFIG_PATH from an external script." }, "server_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "URL for an external connector server (e.g. 'tcp://localhost:5555'). Connectors that run in multi-process mode use this to reach the cache server.", "title": "Server Url" } }, "title": "KvCacheConnectorConfig", "type": "object" }, "KvPoolRebalanceConfig": { "additionalProperties": false, "description": "Tuning for the KV cache manager v2 pool rebalance auto-tuner.\n\nOnly used when ``kv_cache_config.enable_kv_pool_rebalance`` is True.", "properties": { "check_interval": { "default": 10, "description": "Number of PyExecutor iterations between checks of whether a pool rebalance should be attempted.", "exclusiveMinimum": 0, "status": "prototype", "title": "Check Interval", "type": "integer" }, "cooldown_secs": { "default": 120.0, "description": "Minimum time, in seconds, between successive rebalance adjustments.", "minimum": 0, "status": "prototype", "title": "Cooldown Secs", "type": "number" }, "min_sampled_kv_caches": { "default": 2000, "description": "Minimum number of sampled (closed) KvCaches observed before the auto-tuner will consider making an adjustment.", "minimum": 0, "status": "prototype", "title": "Min Sampled Kv Caches", "type": "integer" }, "moving_average_decay": { "default": 0.9999, "description": "Decay factor for the exponential moving averages (reused length, capacity, history length) used to compute target pool ratios. Higher values react more slowly to change.", "exclusiveMaximum": 1, "exclusiveMinimum": 0, "status": "prototype", "title": "Moving Average Decay", "type": "number" }, "ratio_tolerance": { "default": 0.25, "description": "How far a pool group's current ratio may drift from its target before a rebalance is triggered, as a fraction of the smaller of the two. The comparison is symmetric, so 0.25 means a rebalance is triggered once one of the two exceeds the other by more than 25%.", "exclusiveMinimum": 0, "status": "prototype", "title": "Ratio Tolerance", "type": "number" }, "target_ratio_update_interval": { "default": 100, "description": "Number of newly sampled KvCaches between recomputations of the target pool-group ratios.", "exclusiveMinimum": 0, "status": "prototype", "title": "Target Ratio Update Interval", "type": "integer" } }, "title": "KvPoolRebalanceConfig", "type": "object" }, "LayerwiseBenchmarksConfig": { "additionalProperties": false, "description": "Configuration for layer-wise benchmarks calibration.", "properties": { "calibration_file_path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The file path which the layer-wise benchmarks calibrator saves to or loads from", "status": "prototype", "title": "Calibration File Path" }, "calibration_layer_indices": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Layer indices to filter. If None, all layers are collected in COLLECT mode.", "status": "prototype", "title": "Calibration Layer Indices" }, "calibration_mode": { "default": "NONE", "description": "Instruct the layer-wise benchmarks calibrator to work on MARK mode, or COLLECT mode", "enum": [ "NONE", "MARK", "COLLECT" ], "status": "prototype", "title": "Calibration Mode", "type": "string" } }, "title": "LayerwiseBenchmarksConfig", "type": "object" }, "LoadFormat": { "enum": [ 0, 1, 2, 3 ], "title": "LoadFormat", "type": "integer" }, "LoraConfig": { "additionalProperties": false, "properties": { "cuda_graph_specialize_lora": { "default": false, "description": "Whether to capture separate CUDA graphs for batches with and without active LoRA adapters. Reduces overhead for batches without LoRA at the cost of increased warmup time and CUDA graph metadata.", "title": "Cuda Graph Specialize Lora", "type": "boolean" }, "lora_ckpt_source": { "default": "hf", "description": "Checkpoint format: 'hf' for HuggingFace PEFT format, 'nemo' for NeMo format. NeMo only supports attn_qkv module.", "enum": [ "hf", "nemo" ], "title": "Lora Ckpt Source", "type": "string" }, "lora_dir": { "description": "List of directories containing LoRA adapter checkpoints. For HuggingFace format, expects adapter_model.bin/.safetensors and adapter_config.json. For NeMo format, expects .nemo files. If empty, lora_target_modules must be provided.", "items": { "type": "string" }, "title": "Lora Dir", "type": "array" }, "lora_target_modules": { "description": "TensorRT-LLM module names where LoRA is applied (e.g., ['attn_q', 'attn_k', 'attn_v'], ['attn_qkv'], ['mlp_gate', 'mlp_up']). If empty and lora_dir is provided, inferred from checkpoint.", "items": { "type": "string" }, "title": "Lora Target Modules", "type": "array" }, "max_cpu_loras": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of LoRA adapters that can be stored in CPU memory. Controls host-side cache for prefetching adapters before moving to GPU.", "title": "Max Cpu Loras" }, "max_lora_rank": { "default": 64, "description": "Maximum LoRA rank across all adapters, used to pre-allocate workspace memory. Set to the actual max rank of your adapters to reduce memory usage.", "title": "Max Lora Rank", "type": "integer" }, "max_loras": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of LoRA adapters that can be loaded simultaneously in GPU memory. Controls device-side LoRA cache size.", "title": "Max Loras" }, "overlap_lora_and_base": { "default": false, "description": "Whether to place LoRA operations on a secondary CUDA stream and overlap them with base model computations. Improves latency in memory-bound regimes.", "title": "Overlap Lora And Base", "type": "boolean" }, "swap_gate_up_proj_lora_b_weight": { "default": true, "description": "Whether to swap gate/up projection order in fused gate_up_proj LoRA B weights. Set to False for models like Phi-4-MM that use a different weight order.", "title": "Swap Gate Up Proj Lora B Weight", "type": "boolean" }, "trtllm_modules_to_hf_modules": { "additionalProperties": { "type": "string" }, "description": "Mapping from TensorRT-LLM module names to HuggingFace module names (e.g., {'attn_q': 'q_proj'}). If empty, uses default mappings.", "title": "Trtllm Modules To Hf Modules", "type": "object" } }, "title": "LoraConfig", "type": "object" }, "MTPDecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "begin_thinking_phase_token": { "default": 128798, "description": "Token ID marking start of thinking phase. Relaxed acceptance only applies within this phase.", "minimum": 0, "title": "Begin Thinking Phase Token", "type": "integer" }, "decoding_type": { "const": "MTP", "default": "MTP", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "dynamic_tree_max_topK": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Top-K candidates expanded per node per draft layer when use_dynamic_tree is enabled. Required when use_dynamic_tree is True; ignored (with a warning) when use_dynamic_tree is False.", "title": "Dynamic Tree Max Topk" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "end_thinking_phase_token": { "default": 128799, "description": "Token ID marking end of thinking phase. Strict acceptance resumes after this.", "minimum": 0, "title": "End Thinking Phase Token", "type": "integer" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "num_nextn_predict_layers": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Number of MTP layers in the model checkpoint. Auto-populated from the target pretrained config, or from speculative_model's config.json when MTP heads are loaded separately. Do not set manually.", "title": "Num Nextn Predict Layers" }, "relaxed_delta": { "default": 0.0, "description": "Probability threshold for relaxed acceptance. Only candidates with prob >= (top-1 prob - delta) are kept.", "minimum": 0, "title": "Relaxed Delta", "type": "number" }, "relaxed_topk": { "default": 1, "description": "Number of top candidate tokens to consider for relaxed acceptance. Draft token is accepted if it matches any of these.", "exclusiveMinimum": 0, "title": "Relaxed Topk", "type": "integer" }, "sa_config": { "anyOf": [ { "$ref": "#/$defs/SAEnhancerConfig" }, { "type": "null" } ], "default": null, "description": "Optional Suffix Automaton configuration. When set, combines SA drafting with MTP speculative decoding.", "status": "beta" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_dynamic_tree": { "default": false, "description": "Enable EAGLE-style dynamic-tree drafting for one-model MTP. When True, each draft step expands dynamic_tree_max_topK candidates per node and the tree is verified against the target, instead of a linear chain.", "title": "Use Dynamic Tree", "type": "boolean" }, "use_mtp_vanilla": { "default": false, "description": "Force vanilla MTP mode (sequential MTP layers). When False, uses EAGLE-style MTP for single-layer checkpoints.", "title": "Use Mtp Vanilla", "type": "boolean" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" }, "use_relaxed_acceptance_for_thinking": { "default": false, "description": "Enable relaxed acceptance during thinking phase for reasoning models. Accepts draft tokens matching any top-K candidate instead of exact top-1.", "title": "Use Relaxed Acceptance For Thinking", "type": "boolean" } }, "title": "MTPDecodingConfig", "type": "object" }, "MambaStateConfig": { "additionalProperties": false, "description": "Configuration for reusable Mamba recurrent-state snapshots.", "properties": { "additional_snapshot_offsets_from_end": { "description": "Additional Mamba state snapshot offsets measured backward from the end of each prompt. An offset of 0 selects the prompt end. Offsets that do not resolve inside the prompt are ignored. These snapshots require KV cache manager V2.", "items": { "minimum": 0, "type": "integer" }, "status": "prototype", "telemetry": { "exclude": true }, "title": "Additional Snapshot Offsets From End", "type": "array" }, "additional_snapshot_offsets_from_start": { "description": "Additional Mamba state snapshot offsets measured from the start of each prompt. Offsets beyond the prompt length are ignored. These snapshots require KV cache manager V2.", "items": { "exclusiveMinimum": 0, "type": "integer" }, "status": "prototype", "telemetry": { "exclude": true }, "title": "Additional Snapshot Offsets From Start", "type": "array" }, "enable_branch_snapshot": { "default": false, "description": "Snapshot the Mamba recurrent state where a request's content diverges from the prefix cache, so that later requests sharing the same prefix can reuse up to the fork instead of being truncated to an earlier snapshot. If no periodic or additional snapshot point applies to the prompt, the prompt end is snapshotted as a fallback. Requires KV cache manager V2.", "status": "prototype", "telemetry": {}, "title": "Enable Branch Snapshot", "type": "boolean" }, "periodic_snapshot_interval": { "default": 0, "description": "The number of tokens between periodic snapshots in the Mamba prefix cache. Periodic snapshots are disabled by default; set this to a positive value to enable them.", "minimum": 0, "status": "prototype", "telemetry": {}, "title": "Periodic Snapshot Interval", "type": "integer" } }, "title": "MambaStateConfig", "type": "object" }, "MiniMaxM3SparseAttentionConfig": { "additionalProperties": false, "description": "Configuration for MiniMax-M3 block-sparse attention.\n\nDrives the two-step sparse attention used by MiniMax-M3 layers 3..N:\n\n 1. An index attention branch projects a per-head Q vector and a single\n replicated K vector, scores main K/V cache blocks, and selects the\n top-k blocks per (num_kv_heads, q_token) pair, with init_blocks forced\n at the head and local_blocks forced at the tail.\n 2. A sparse GQA attention runs only over the selected blocks.\n\nAt runtime one of the MiniMax-M3 sparse attention backends under\ntensorrt_llm._torch.attention.backends.sparse.minimax_m3 is selected. The\nchosen backend runs on top of a MiniMaxM3KVCacheManagerV2 that allocates a\npaged side index-K cache of shape [num_slots, 1, sparse_index_dim] parallel\nto the main K/V cache. The M3 checkpoint sets disable_index_value=True on\nevery sparse layer, so no index V cache is allocated.", "properties": { "algorithm": { "const": "minimax_m3", "default": "minimax_m3", "title": "Algorithm", "type": "string" }, "fuse_qkv_index_projection": { "default": false, "description": "Fuse Q/K/V and index-Q/index-K into one quantized projection. Index-Q is sharded with the KV heads and index-K is replicated. MSA batches also use a horizontal norm/RoPE/cache-insertion producer for prefill, mixed, and CUDA-graph decode execution. The MiniMax-M3-specific path requires the MSA implementation, indexer_kv_dtype='fp8', and an FP8 or NVFP4 main KV cache.", "status": "prototype", "title": "Fuse Qkv Index Projection", "type": "boolean" }, "implementation": { "default": "triton", "description": "Sparse attention implementation: 'triton' legacy reference (default) or the recommended 'msa' backend. MSA requires an SM100 or SM103 GPU, the fmha_sm100 package, and sparse_block_size == 128. NVFP4 KV cache requires 'msa'; it uses MSA prefill and Triton sparse decode kernels.", "enum": [ "triton", "msa" ], "status": "prototype", "title": "Implementation", "type": "string" }, "indexer_kv_dtype": { "default": "bf16", "description": "Storage and score-compute dtype for normalized index Q/K. 'fp8' uses unscaled E4M3 values with FP32 score accumulation and is supported only by the MSA implementation.", "enum": [ "bf16", "fp8" ], "status": "prototype", "title": "Indexer Kv Dtype", "type": "string" }, "num_attention_heads": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Global number of attention (query) heads. When unset, it falls back to pretrained_config.num_attention_heads.", "title": "Num Attention Heads" }, "num_key_value_heads": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Global number of key/value heads. When unset, it falls back to pretrained_config.num_key_value_heads, then to num_attention_heads.", "title": "Num Key Value Heads" }, "sparse_block_size": { "default": 128, "description": "Block size used by per-block scoring + top-k selection.", "title": "Sparse Block Size", "type": "integer" }, "sparse_disable_index_value": { "default": true, "description": "If True, skip the index V branch (M3 checkpoint default).", "title": "Sparse Disable Index Value", "type": "boolean" }, "sparse_index_dim": { "default": 128, "description": "Per-head index Q/K dimension.", "exclusiveMinimum": 0, "title": "Sparse Index Dim", "type": "integer" }, "sparse_init_blocks": { "default": 0, "description": "Number of leading blocks forced into the top-k regardless of score.", "title": "Sparse Init Blocks", "type": "integer" }, "sparse_local_blocks": { "default": 1, "description": "Number of trailing blocks forced into the top-k regardless of score.", "title": "Sparse Local Blocks", "type": "integer" }, "sparse_num_index_heads": { "default": 4, "description": "Global checkpoint index-attention head count. Index heads shard with their KV-head groups in both separate and fused projections.", "exclusiveMinimum": 0, "title": "Sparse Num Index Heads", "type": "integer" }, "sparse_score_type": { "const": "max", "default": "max", "description": "Per-block score reduction; the M3 checkpoint sets 'max'.", "title": "Sparse Score Type", "type": "string" }, "sparse_topk_blocks": { "default": 16, "description": "Number of top-k blocks per (kv_head, q_token).", "title": "Sparse Topk Blocks", "type": "integer" } }, "title": "MiniMaxM3SparseAttentionConfig", "type": "object" }, "MinimalInstances": { "additionalProperties": false, "properties": { "context_servers": { "default": 1, "title": "Context Servers", "type": "integer" }, "generation_servers": { "default": 1, "title": "Generation Servers", "type": "integer" } }, "title": "MinimalInstances", "type": "object" }, "ModelExpressConfig": { "additionalProperties": false, "description": "Prototype configuration for ModelExpress (MX) weight transfer.", "properties": { "preshard_strategy": { "default": "per_module", "description": "How to inform TRT-LLM that MX-delivered weights are already TP-sharded for the local rank. Only 'per_module' is supported in this MX-only PR; 'global' requires LoadFormat.PRESHARDED.", "status": "prototype", "telemetry": { "allowed_values": [ "per_module" ] }, "title": "Preshard Strategy", "type": "string" }, "server_query_timeout_s": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "deprecated": true, "description": "Deprecated and ignored. ModelExpress performs one source discovery query and falls back to native checkpoint loading when no compatible source is immediately available.", "status": "deprecated", "title": "Server Query Timeout S" }, "server_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "URL of the MX (ModelExpress) server for P2P weight transfer. When set together with checkpoint_format='MX', enables GPU-to-GPU weight transfer from a running source instance, bypassing disk I/O. When the server is unreachable, loading falls back to the standard HuggingFace checkpoint path.", "status": "prototype", "title": "Server Url" } }, "title": "ModelExpressConfig", "type": "object" }, "MoeConfig": { "additionalProperties": false, "description": "Configuration for MoE.", "properties": { "backend": { "default": "AUTO", "description": "MoE backend to use. AUTO selects default backend based on model. It currently doesn't always give the best choice for all scenarios. The capabilities of auto selection will be improved in future releases.", "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "title": "Backend", "type": "string" }, "disable_finalize_fusion": { "default": false, "description": "Disable FC2+finalize kernel fusion in CUTLASS MoE backend. Setting this to True recovers deterministic numerical behavior with top-k > 2.", "title": "Disable Finalize Fusion", "type": "boolean" }, "load_balancer": { "anyOf": [ { "additionalProperties": false, "description": "Pydantic configuration model for the Mixture of Experts (MoE) load balancer.\n\nThis model holds configuration data (`num_slots`, etc.) as well as\nruntime state (`_ep_rank`, `_ep_size`) which must be set via the\n`setup()` method before use.", "properties": { "initial_global_assignments": { "anyOf": [ { "additionalProperties": { "items": { "type": "integer" }, "type": "array" }, "type": "object" }, { "type": "null" } ], "default": null, "title": "Initial Global Assignments" }, "layer_updates_per_iter": { "default": 0, "title": "Layer Updates Per Iter", "type": "integer" }, "num_slots": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "title": "Num Slots" } }, "title": "MoeLoadBalancerConfig", "type": "object" }, { "type": [ "string", "null" ] } ] }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "If set, at most max_num_tokens tokens will be sent to torch.ops.trtllm.fused_moe at the same time. If the number of tokens exceeds max_num_tokens, the input tensors will be split into chunks and a for loop will be used.", "title": "Max Num Tokens" }, "use_low_precision_moe_combine": { "default": false, "description": "Use low precision combine in MoE operations (only for NVFP4 quantization). When enabled, uses lower precision for combining expert outputs to improve performance.", "title": "Use Low Precision Moe Combine", "type": "boolean" } }, "title": "MoeConfig", "type": "object" }, "MooncakeStoreConfig": { "additionalProperties": false, "description": "How this server joins a Mooncake store pool.\n\nThe settings every participant shares come from the master named by\n`pool`; what is left here is per-server. Setting this makes\n`trtllm-serve` render the Mooncake client config and export\n`MOONCAKE_CONFIG_PATH`; an inherited `MOONCAKE_CONFIG_PATH` wins.", "properties": { "local_hostname": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Address this server's ranks register their segments under, for peers to reach them at. Derived by default from the interface that routes to the master. One value covers every rank, so only set it on a server whose ranks share a node.", "telemetry": { "exclude": true }, "title": "Local Hostname" }, "master_timeout": { "default": 60.0, "description": "Seconds to wait for the pool manifest to appear and the master to accept connections. Raise it when the wait spans a container start on another node. Expiring fails the server at startup.", "telemetry": { "exclude": true }, "title": "Master Timeout", "type": "number" }, "model_key": { "description": "What the pool keys identify this checkpoint by. Two engines share cache only when they agree on it, and two that disagree read each other's pages as their own, so it has no default and must be unique per checkpoint.", "telemetry": { "exclude": true }, "title": "Model Key", "type": "string" }, "namespace": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Key namespace, isolating this deployment's cache from others on the same pool. Bump it after any change to page layout or contents. Defaults to the pool manifest's.", "title": "Namespace" }, "pool": { "description": "The pool to join: 'file://' naming a manifest published by 'trtllm-serve mooncake_master --pool_file', or a master's 'host:port'. The manifest form also carries the settings every participant must agree on.", "title": "Pool", "type": "string" }, "role": { "default": "both", "description": "What this server does with the pool. 'both' reads and writes, 'producer' only writes, 'consumer' only reads, and 'capacity' does neither: its ranks lend memory without moving any KV, so they pin no staging buffers and pay no per-request cost.", "enum": [ "both", "producer", "consumer", "capacity" ], "title": "Role", "type": "string" }, "run_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Where this server keeps the Mooncake client config it renders and each rank's record of the segment it mounted. Required when a launcher starts one task per rank, as trtllm-llmapi-launch does, and must not be shared between servers. Defaults to a temporary directory removed at shutdown.", "telemetry": { "exclude": true }, "title": "Run Dir" }, "segment_size": { "anyOf": [ { "type": "integer" }, { "type": "string" } ], "default": "16GiB", "description": "Host memory each of this server's ranks contributes to the pool, as a binary size ('16GiB') or a byte count; ambiguous units such as 'GB' are refused. A node's demand is ranks_on_node x segment_size, checked against available memory at startup. Zero lends nothing, leaving the server to use only capacity its peers hold.", "title": "Segment Size" }, "stage_through_host": { "default": true, "description": "Copy pages through a pinned host buffer instead of registering the KV pools with Mooncake. The only transfer path supported today, so 'false' is ignored with a warning. Costs a copy each way and needs no GPUDirect RDMA.", "telemetry": { "exclude": true }, "title": "Stage Through Host", "type": "boolean" }, "transfer_batch_size": { "default": 64, "description": "Page keys per store call.", "exclusiveMinimum": 0, "telemetry": { "exclude": true }, "title": "Transfer Batch Size", "type": "integer" } }, "required": [ "pool", "model_key" ], "title": "MooncakeStoreConfig", "type": "object" }, "MultimodalConfig": { "additionalProperties": false, "description": "Multimodal model configuration.", "properties": { "encoder_cache_max_bytes": { "default": 134217728, "description": "Target capacity in bytes for reusable multimodal encoder embeddings; 0 disables cross-request reuse. The capacity applies per rank; ranks do not share a cache. With item scheduling, encoder outputs still use the model-owned cache and its capacity is at least one legal encoder iteration; this value can make that cache larger. String values such as '512MB' and '1GiB' use binary units. Both inline encoding and item scheduling cache individual items; inline caching supports only single-modality parameters. Compatible with side-stream prefetch.", "minimum": 0, "status": "prototype", "title": "Encoder Cache Max Bytes", "type": "integer" }, "encoder_cuda_graph": { "anyOf": [ { "additionalProperties": { "$ref": "#/$defs/MultimodalEncoderCudaGraphConfig" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "CUDA graph capture for multimodal encoders, keyed by modality name. This config is not applied automatically - each model must read `model_config.multimodal_config.encoder_cuda_graph` and implement capture + replay via `MultimodalEncoderCudaGraphRunner` (see `tensorrt_llm/_torch/models/multimodal_encoder_graph.py`).", "status": "prototype", "title": "Encoder Cuda Graph" }, "encoder_scheduling_policy": { "$ref": "#/$defs/MultimodalEncoderSchedulingPolicy", "default": "DEFAULT", "description": "MM encoder scheduling policy for models that support item-level encoder scheduling. DISABLED: legacy inline encode (item scheduling and its byte budget off). DEFAULT: item scheduling. EAGER: item scheduling that advances encoder work for active requests before LLM capacity filtering. Ignored for models that do not support item scheduling. Item scheduling does not support pipeline parallelism yet: with pipeline_parallel_size > 1, DEFAULT (explicit or unset) logs a warning and falls back to legacy inline encode, while EAGER raises an error.", "status": "prototype" }, "encoder_side_stream_max_ahead": { "default": 0, "description": "Maximum number of pending multimodal requests whose encoder work can be prefetched on a side CUDA stream ahead of admission. 0 disables side-stream prefetch. Incompatible with encoder_cuda_graph because graph replay uses static buffers. Can be combined with encoder_cache_max_bytes; the two memory limits are additive.", "minimum": 0, "status": "prototype", "title": "Encoder Side Stream Max Ahead", "type": "integer" }, "video_pruning_rate": { "anyOf": [ { "exclusiveMaximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Pruning rate for video frames in multimodal models for Efficient Video Sampling (EVS). NOTE: this is currently only implemented in nemotron multimodal models. None (default) disables EVS, values in [0, 1) enable pruning.", "status": "prototype", "title": "Video Pruning Rate" } }, "title": "MultimodalConfig", "type": "object" }, "MultimodalEncoderCudaGraphConfig": { "additionalProperties": false, "description": "CUDA graph capture for multimodal vision / audio encoders.\n\nOne graph is captured per `buckets` entry; replay falls back to eager when the live workload's\npadded shape does not match any bucket.\n\nNOTE: enabling this will lead to higher GPU memory usage. It is usually more beneficial for\n`buckets` corresponding to lesser compute.", "properties": { "buckets": { "description": "Explicit encoder graph buckets as `(total_tokens, num_contexts)` pairs. Each pair captures one graph sized for up to `total_tokens` real encoder tokens across exactly `num_contexts` real contexts.", "items": { "maxItems": 2, "minItems": 2, "prefixItems": [ { "exclusiveMinimum": 0, "type": "integer" }, { "exclusiveMinimum": 0, "type": "integer" } ], "type": "array" }, "minItems": 1, "status": "prototype", "title": "Buckets", "type": "array" }, "enable_padding": { "default": true, "description": "Capture a padded variant of each bucket so partial buckets can replay. Padding works only when the request has exactly the bucket's `num_contexts` real contexts and no more than the bucket's `total_tokens` real encoder tokens. The runner appends one dummy padding context and one reserved padding token internally, so bucket values should describe only the real workload.", "status": "prototype", "title": "Enable Padding", "type": "boolean" }, "enable_replay_stats": { "default": false, "description": "Log diagnostic details for encoder CUDA graph bucket hits and misses. When enabled, each request's bucket decision is logged at INFO.", "status": "prototype", "title": "Enable Replay Stats", "type": "boolean" }, "warmup_steps": { "default": 2, "description": "Warmup iterations to run per bucket before capturing.", "exclusiveMinimum": 0, "status": "prototype", "title": "Warmup Steps", "type": "integer" } }, "required": [ "buckets" ], "title": "MultimodalEncoderCudaGraphConfig", "type": "object" }, "MultimodalEncoderSchedulingPolicy": { "description": "Selects how a model that supports item-level MM encoder scheduling runs its encoder.\n\nIgnored for models that do not support it (they always use legacy inline\nencode).", "enum": [ "DISABLED", "DEFAULT", "EAGER" ], "title": "MultimodalEncoderSchedulingPolicy", "type": "string" }, "NGramDecodingConfig": { "additionalProperties": false, "description": "Configuration for NGram drafter speculative decoding.", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "NGram", "default": "NGram", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "is_keep_all": { "default": true, "description": "Whether to keep all candidate pattern-matches pairs, only one match is kept for each pattern if False.", "title": "Is Keep All", "type": "boolean" }, "is_public_pool": { "default": true, "description": "Whether to use a common pool for all requests, or the pool is private for each request if False.", "title": "Is Public Pool", "type": "boolean" }, "is_use_oldest": { "default": true, "description": "Whether to provide the oldest match when pattern is hit, the newest one is provided if False.", "title": "Is Use Oldest", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_matching_ngram_size": { "default": 2, "description": "The length maximum of searching tokens (can be understood as length maximum of input tokens to search).", "exclusiveMinimum": 0, "title": "Max Matching Ngram Size", "type": "integer" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "NGramDecodingConfig", "type": "object" }, "Nvfp4GemmConfig": { "additionalProperties": false, "description": "Configuration for NVFP4 GEMM backend selection.", "properties": { "allowed_backends": { "description": "List of backends to consider for auto-selection. Default excludes 'cutedsl' for faster build time. Add 'cutedsl' for extreme performance at the cost of longer server launch time.", "items": { "enum": [ "cutlass", "cublaslt", "cutedsl", "cuda_core", "marlin" ], "type": "string" }, "minItems": 1, "title": "Allowed Backends", "type": "array" } }, "title": "Nvfp4GemmConfig", "type": "object" }, "PARDDecodingConfig": { "additionalProperties": false, "description": "Configuration for PARD (Parallel Draft) speculative decoding.\n\nPARD is a target-independent speculative decoding method that uses\nmask tokens to predict multiple draft tokens in parallel within a\nsingle forward pass.\n\nKey features:\n- Target-independent: doesn't use target model hidden states\n- Parallel prediction: all K draft tokens in one forward pass\n- Shared mask token: uses the same mask_token_id across all positions\n\nReference: https://arxiv.org/pdf/2504.18583", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "PARD", "default": "PARD", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "mask_token_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The token ID used as a mask token for parallel draft prediction. If None, it will be read from the draft model config (typically vocab_size).", "title": "Mask Token Id" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "sa_config": { "anyOf": [ { "$ref": "#/$defs/SAEnhancerConfig" }, { "type": "null" } ], "default": null, "description": "Optional Suffix Automaton configuration. When set, combines SA drafting with PARD speculative decoding.", "status": "beta" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "PARDDecodingConfig", "type": "object" }, "PeftCacheConfig": { "additionalProperties": false, "description": "Configuration for the PEFT cache.", "properties": { "device_cache_percent": { "default": 0.02, "description": "Proportion of free device memory after engine load to use for cache, as a fraction from 0 to 1", "title": "Device Cache Percent", "type": "number" }, "host_cache_size": { "default": 1073741824, "description": "size in bytes to use for host cache", "title": "Host Cache Size", "type": "integer" }, "lora_prefetch_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "folder to store the LoRA weights we hope to load during engine initialization, currently not supported", "title": "Lora Prefetch Dir" }, "max_adapter_size": { "default": 64, "description": "max supported adapter size. Used to compute minimum", "title": "Max Adapter Size", "type": "integer" }, "max_pages_per_block_device": { "default": 8, "description": "Number of cache pages per allocation block (device)", "title": "Max Pages Per Block Device", "type": "integer" }, "max_pages_per_block_host": { "default": 24, "description": "Number of cache pages per allocation block (host)", "title": "Max Pages Per Block Host", "type": "integer" }, "num_copy_streams": { "default": 1, "description": "number of streams used to copy weights from host to device", "title": "Num Copy Streams", "type": "integer" }, "num_device_module_layer": { "default": 0, "description": "number of max sized 1-layer 1-module sets of weights that can be stored in device cache, affects device cache size and overrides value of device_cache_percent", "title": "Num Device Module Layer", "type": "integer" }, "num_ensure_workers": { "default": 1, "description": "number of worker threads used to copy weights from host to device", "title": "Num Ensure Workers", "type": "integer" }, "num_host_module_layer": { "default": 0, "description": "number of max sized 1-layer 1-module adapterSize=1 sets of weights that can be stored in host cache, affects host cache size and overrides value of host_cache_size", "title": "Num Host Module Layer", "type": "integer" }, "num_put_workers": { "default": 1, "description": "number of worker threads used to put weights into host cache", "title": "Num Put Workers", "type": "integer" }, "optimal_adapter_size": { "default": 8, "description": "optimal adapter size used to set page width", "title": "Optimal Adapter Size", "type": "integer" } }, "title": "PeftCacheConfig", "type": "object" }, "PrefillCudaGraphBackend": { "description": "CUDA graph implementation used for prefill requests.", "enum": [ "disabled", "piecewise", "breakable" ], "title": "PrefillCudaGraphBackend", "type": "string" }, "PrometheusMetricsConfig": { "additionalProperties": false, "description": "Configuration for Prometheus metrics collection.\n\nGroups all Prometheus-related parameters including custom histogram bucket\nboundaries for latency metrics.", "properties": { "e2e_request_latency_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_e2e_request_latency_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "E2E Request Latency Buckets" }, "request_decode_time_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_request_decode_time_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Request Decode Time Buckets" }, "request_inference_time_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_request_inference_time_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Request Inference Time Buckets" }, "request_prefill_time_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_request_prefill_time_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Request Prefill Time Buckets" }, "request_queue_time_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_request_queue_time_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Request Queue Time Buckets" }, "time_per_output_token_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_time_per_output_token_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Time Per Output Token Buckets" }, "time_to_first_token_buckets": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Custom histogram bucket boundaries (in seconds) for trtllm_time_to_first_token_seconds. Defaults to built-in values when unset.", "status": "prototype", "title": "Time To First Token Buckets" } }, "title": "PrometheusMetricsConfig", "type": "object" }, "QSASparseAttentionConfig": { "additionalProperties": false, "description": "Configuration for QSA compressed query-selected attention.", "properties": { "algorithm": { "const": "qsa", "default": "qsa", "description": "Select QSA compressed query-selected sparse attention.", "title": "Algorithm", "type": "string" }, "enable_heuristic_topk": { "default": false, "description": "Whether to enable the Guess-Verify-Refine (GVR) Top-K for the QSA indexer instead of the exact radix Top-K. QSA dispatches only the hint-free self-sampling engine, which requires Blackwell (SM100/103), the CUTLASS DSL, a compressed-group budget (indexer_budget / indexer_compress_ratio) in {512, 1024, 2048}, and an indexer_compress_ratio of 4. Falls back to the exact radix Top-K with a one-time warning when the prerequisites are not met.", "title": "Enable Heuristic Topk", "type": "boolean" }, "index_share_for_mtp_iteration": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "Whether the MTP draft loop reuses the indexer selection captured by its draft-extend pass instead of re-running the indexer on every draft decode step. The query advances by at most max_draft_len positions, so the captured ranking is the one the indexer would recompute; compressed groups that complete during the loop are appended at lookup. When omitted, the checkpoint config supplies the value, defaulting to off.", "status": "prototype", "title": "Index Share For Mtp Iteration" }, "seq_len_threshold": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The sequence length threshold separating dense and QSA attention. When omitted, it is resolved to token_topk after checkpoint geometry is loaded; an explicit value below token_topk is raised to token_topk because it cannot reduce attention work.", "title": "Seq Len Threshold" } }, "title": "QSASparseAttentionConfig", "type": "object" }, "RayPlacementConfig": { "additionalProperties": false, "description": "Configuration for Ray GPU workers placement.\nCurrently, this config is only used with AsyncLLM for RL scenarios.", "properties": { "defer_workers_init": { "default": false, "description": "Defer Ray worker initialization until async setup.", "title": "Defer Workers Init", "type": "boolean" }, "per_worker_gpu_share": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "description": "GPU fraction per worker for colocation scenarios. Example: 0.1 means 10 actors can share one GPU. Defaults to 1.0 (one actor per GPU).", "title": "Per Worker Gpu Share" }, "placement_bundle_indices": { "anyOf": [ { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of lists of bundle indices. The outer list corresponds to `placement_groups`. Each inner list specifies the bundle indices to use within that placement group. For example, if `placement_groups=[pg1, pg2]`, `[[0, 1], [0, 1]]` assigns bundles 0 and 1 from `pg1` and bundles 0 and 1 from `pg2`.", "title": "Placement Bundle Indices" }, "placement_groups": { "type": "null" } }, "title": "RayPlacementConfig", "type": "object" }, "ReorderRequestPolicyConfig": { "additionalProperties": false, "description": "Configuration for request reordering policy.", "properties": { "policy_args": { "$ref": "#/$defs/AgentTreeConfig", "description": "The arguments of the request reordering policy." }, "policy_name": { "anyOf": [ { "const": "AgentTree", "type": "string" }, { "type": "null" } ], "default": null, "description": "The name of the request reordering policy.", "title": "Policy Name" } }, "title": "ReorderRequestPolicyConfig", "type": "object" }, "RocketSparseAttentionConfig": { "additionalProperties": false, "description": "Configuration for RocketKV sparse attention.", "properties": { "algorithm": { "const": "rocket", "default": "rocket", "title": "Algorithm", "type": "string" }, "kernel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 63, "description": "The kernel size for RocketKV.", "title": "Kernel Size" }, "kt_cache_dtype": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": "float8_e5m2", "description": "KT cache dtype", "telemetry": { "allowed_values": [ "bfloat16", "float8_e5m2" ] }, "title": "Kt Cache Dtype" }, "page_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 4, "description": "Page size", "title": "Page Size" }, "prompt_budget": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 2048, "description": "Prompt budget", "title": "Prompt Budget" }, "seq_len_threshold": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The sequence length threshold for separating short and long sequences.", "title": "Seq Len Threshold" }, "topk": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 64, "description": "Top-k", "title": "Topk" }, "topr": { "anyOf": [ { "type": "integer" }, { "type": "number" }, { "type": "null" } ], "default": 128, "description": "Top-r", "title": "Topr" }, "window_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 32, "description": "The window size for RocketKV.", "title": "Window Size" } }, "title": "RocketSparseAttentionConfig", "type": "object" }, "SADecodingConfig": { "additionalProperties": false, "description": "Configuration for standalone Suffix Automaton (SA) speculative decoding.\n\nUses a GPU-native suffix automaton for pattern matching. Drafting runs\ninside the target model forward; supports CUDA graph and overlap scheduler.\n\nTo combine SA with a neural drafter (Eagle3, MTP, PARD) instead of using\nit standalone, pass :class:`SAEnhancerConfig` via ``sa_config``.", "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "SA", "default": "SA", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "enable_global_pool": { "default": false, "description": "When True, each request searches all active SA states for the longest match, not just its own. Improves acceptance rates when requests share common patterns. Limitations: at most 1024 concurrent slots; suffix matching is capped at 64 tokens per request.", "title": "Enable Global Pool", "type": "boolean" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "global_pool_size": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "Number of SA slots in the global pool. When None and enable_global_pool=True, defaults to max(64, max_batch_size) \u2014 a fixed-size pool independent of batch size. When set explicitly, must be >= max_batch_size. Completed requests' SA states are retained in the pool for cross-request search until the pool is full, at which point the oldest completed request is evicted. Only effective when enable_global_pool=True.", "title": "Global Pool Size" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_matching_ngram_size": { "default": -1, "description": "Positive value (e.g., 3): fixed-size ngram matching. -1: longest possible match via suffix automaton. 0 is invalid.", "title": "Max Matching Ngram Size", "type": "integer" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "title": "SADecodingConfig", "type": "object" }, "SAEnhancerConfig": { "additionalProperties": false, "description": "Configuration for the Suffix Automaton (SA) draft enhancer.\n\nUse this to combine SA pattern-matching drafting with another speculative\ndecoding method (Eagle3, MTP, PARD). When provided as ``sa_config`` on a\ndecoding config, SA drafting is enabled and may override neural draft\ntokens when the suffix match length meets the *threshold*.\n\nFor standalone SA speculative decoding (no neural drafter), use\n:class:`SADecodingConfig` instead.", "properties": { "enable_global_pool": { "default": false, "description": "When True, each request searches all active SA states for the longest match, not just its own. Improves acceptance rates when requests share common patterns.", "title": "Enable Global Pool", "type": "boolean" }, "threshold": { "default": 4, "description": "Minimum suffix match length required for the SA output to override neural draft tokens.", "exclusiveMinimum": 0, "title": "Threshold", "type": "integer" } }, "title": "SAEnhancerConfig", "type": "object" }, "SaveHiddenStatesDecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "SaveState", "default": "SaveState", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "eagle3_layers_to_capture": { "anyOf": [ { "items": { "type": "integer" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "default": null, "description": "Set of target model layer indices to capture hidden states from for EAGLE3 draft model training. Use -1 to indicate the final post-norm hidden state. If not provided, defaults to capturing 3 intermediate layers plus the post-norm hidden state. When provided, -1 is automatically added if not present.", "title": "Eagle3 Layers To Capture" }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "file_prefix": { "default": "data", "description": "Prefix for output filenames. Files are saved as '_.pt' containing input_ids and hidden_state tensors.", "title": "File Prefix", "type": "string" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 1, "description": "Internal field, not user-configurable. Fixed to 1 since this mode captures hidden states without draft token generation.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "output_directory": { "description": "Directory path where hidden states data files will be saved. The directory is created if it does not exist.", "title": "Output Directory", "type": "string" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" }, "write_interval": { "default": 20, "description": "Number of requests to process before writing accumulated hidden states to disk. Lower values write more frequently but may impact performance.", "title": "Write Interval", "type": "integer" } }, "required": [ "output_directory" ], "title": "SaveHiddenStatesDecodingConfig", "type": "object" }, "SchedulerConfig": { "additionalProperties": false, "properties": { "capacity_scheduler_policy": { "$ref": "#/$defs/CapacitySchedulerPolicy", "default": "GUARANTEED_NO_EVICT", "description": "The capacity scheduler policy to use" }, "context_chunking_policy": { "anyOf": [ { "$ref": "#/$defs/ContextChunkingPolicy" }, { "type": "null" } ], "default": null, "description": "The context chunking policy to use" }, "dynamic_batch_config": { "anyOf": [ { "$ref": "#/$defs/DynamicBatchConfig" }, { "type": "null" } ], "default": null, "description": "The dynamic batch config to use. This only applies for the TensorRT backend and cannot currently be used with the PyTorch backend." }, "enable_prefix_aware_scheduling": { "default": true, "description": "Use KV prefix-reuse estimates for scheduler admission, duplicate-request deferral, and token-budget decisions. This is orthogonal to kv_cache_config.enable_block_reuse.", "title": "Enable Prefix Aware Scheduling", "type": "boolean" }, "use_python_scheduler": { "default": false, "description": "Use pure-Python scheduler instead of C++ scheduler.", "title": "Use Python Scheduler", "type": "boolean" }, "waiting_queue_policy": { "$ref": "#/$defs/WaitingQueuePolicy", "default": "fcfs", "description": "The waiting queue scheduling policy" } }, "title": "SchedulerConfig", "type": "object" }, "ServerRole": { "enum": [ 0, 1, 2, 3, 4 ], "title": "ServerRole", "type": "integer" }, "SkipSoftmaxAttentionConfig": { "additionalProperties": false, "description": "Configuration for skip softmax attention.", "properties": { "algorithm": { "const": "skip_softmax", "default": "skip_softmax", "title": "Algorithm", "type": "string" }, "target_sparsity": { "anyOf": [ { "type": "number" }, { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Target sparsity for prefill and/or decode phases. Requires formula coefficients in the model's config.json. Ignored if threshold_scale_factor is also set.", "title": "Target Sparsity" }, "threshold_scale_factor": { "anyOf": [ { "type": "number" }, { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "The threshold scale factor for skip softmax attention.", "title": "Threshold Scale Factor" }, "uses_spcompress": { "default": false, "description": "Whether to enable spcompress (context phase, SM107 only).", "title": "Uses Spcompress", "type": "boolean" } }, "title": "SkipSoftmaxAttentionConfig", "type": "object" }, "SleepConfig": { "additionalProperties": false, "description": "Configuration for the LLM sleep/wakeup feature.", "properties": { "restore_modes": { "description": "Per-component RestoreMode for the sleep feature. Keys are ExecutorMemoryType values (e.g. 'model', 'kv_cache'), values can be RestoreMode names (NONE, MEMSET, CPU, PINNED) or RestoreMode enum values. Unlisted entries default to the suitable mode selected between PINNED and CPU.", "title": "Restore Modes" } }, "title": "SleepConfig", "type": "object" }, "TelemetryConfig": { "additionalProperties": false, "description": "Telemetry configuration for usage data collection.\n\nControls opt-out behavior and tracks which entry point invoked TRT-LLM.", "properties": { "disabled": { "default": false, "description": "Disable anonymous usage telemetry collection. Can also be set via TRTLLM_NO_USAGE_STATS=1, TELEMETRY_DISABLED=true, DO_NOT_TRACK=1, or file ~/.config/trtllm/do_not_track.", "title": "Disabled", "type": "boolean" }, "usage_context": { "$ref": "#/$defs/UsageContext", "default": "unknown", "description": "Identifies how TRT-LLM was invoked (CLI command vs Python API). Set automatically by CLI commands; defaults to UNKNOWN (promoted to LLM_CLASS by BaseLLM.__init__ for direct Python API usage)." } }, "title": "TelemetryConfig", "type": "object" }, "TorchCompileConfig": { "additionalProperties": false, "description": "Configuration for torch.compile.", "properties": { "capture_num_tokens": { "anyOf": [ { "items": { "exclusiveMinimum": 0, "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Deprecated. Use prefill_capture_num_tokens instead.", "title": "Capture Num Tokens" }, "compile_only_piecewise_graphs": { "default": false, "description": "Compile only forwards eligible for piecewise prefill CUDA graphs, while generation-only forwards, prefill graph misses, and auxiliary kernel warmup forwards remain eager. Enabling this will speed up startup, but might lead to degraded performance or inconsistent output in specific configurations.", "status": "prototype", "title": "Compile Only Piecewise Graphs", "type": "boolean" }, "enable_fullgraph": { "default": true, "description": "Enable full graph compilation in torch.compile.", "title": "Enable Fullgraph", "type": "boolean" }, "enable_inductor": { "default": false, "description": "Enable inductor backend in torch.compile.", "title": "Enable Inductor", "type": "boolean" }, "enable_piecewise_cuda_graph": { "default": false, "description": "Deprecated. Use prefill_cuda_graph_backend='piecewise' instead.", "title": "Enable Piecewise Cuda Graph", "type": "boolean" }, "enable_userbuffers": { "default": true, "description": "When torch compile is enabled, userbuffers is enabled by default.", "title": "Enable Userbuffers", "type": "boolean" }, "max_num_streams": { "default": 3, "description": "The maximum number of CUDA streams to use for torch.compile.", "exclusiveMinimum": 0, "title": "Max Num Streams", "type": "integer" } }, "title": "TorchCompileConfig", "type": "object" }, "TriAttentionKvCacheCompressionConfig": { "additionalProperties": false, "description": "TriAttention KV-cache compression: periodic decode-time eviction.\n\nScored by offline calibration (github.com/WeianMao/triattention; supply\nthe official .pt via ``calibration_path``). Pure compression \u2014 decode\nruns the model's standard attention over the compacted cache.", "properties": { "algorithm": { "const": "triattention", "default": "triattention", "title": "Algorithm", "type": "string" }, "beta": { "default": 128, "description": "Eviction period in confirmed generation tokens (upstream `divide_length`): one speculative iteration may advance the counter by multiple accepted tokens; at most one eviction is coalesced per update.", "exclusiveMinimum": 0, "title": "Beta", "type": "integer" }, "budget": { "default": 2048, "description": "Tokens kept at each periodic eviction; prompt tokens are always preserved on top.", "exclusiveMinimum": 0, "title": "Budget", "type": "integer" }, "calibration_path": { "description": "Path to the official TriAttention calibration `.pt` (produced by github.com/WeianMao/triattention). TRT-LLM does not compute calibration; it converts this file to the runtime schema at load.", "minLength": 1, "title": "Calibration Path", "type": "string" }, "eviction_mode": { "default": "union", "description": "Which token set each eviction round keeps. `union` (default) takes the union of each KV head's top-B and re-ranks it by the per-token max score; it matches the official base setting (per-head and per-layer-per-head pruning both off). `per_head` keeps a per-KV-head set shared across layers (mean of per-layer max); `per_layer_perhead` keeps a fully independent set per (layer, KV head).", "enum": [ "union", "per_head", "per_layer_perhead" ], "title": "Eviction Mode", "type": "string" }, "normalize_scores": { "default": true, "description": "Z-normalize each head's scores over the decode region before selection (upstream default). `union` eviction requires True: its fused score+stats+union pipeline always normalizes.", "title": "Normalize Scores", "type": "boolean" } }, "required": [ "calibration_path" ], "title": "TriAttentionKvCacheCompressionConfig", "type": "object" }, "UsageContext": { "description": "Identifies how TRT-LLM was invoked for telemetry tracking.", "enum": [ "unknown", "llm_class", "cli_serve", "cli_bench", "cli_eval", "disaggregated" ], "title": "UsageContext", "type": "string" }, "UserProvidedDecodingConfig": { "additionalProperties": false, "properties": { "acceptance_rate_threshold": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "The threshold for average true acceptance rate (accepted_draft_tokens / drafted_tokens); speculation will be disabled permanently once the rolling average over the last N speculation-enabled decoding iterations (N = acceptance_rate_window_size) drops below this value. ", "title": "Acceptance Rate Threshold" }, "acceptance_rate_window_size": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The rolling average window size (N) for acceptance rate across speculation-enabled decoding iterations. If not set or set to 0, the feature is disabled. PyTorch backend only.", "title": "Acceptance Rate Window Size" }, "advanced_sampling_mode": { "$ref": "#/$defs/AdvancedSamplingMode", "default": "full", "description": "Deploy-time specialization of the one-model advanced sampler that skips disabled filter kernels. FULL (default): per-row top_k/top_p/min_p in one fused kernel, the only mode accepting min_p. NO_TOPK: skip top_k. NO_TOPP: skip top_p. NO_TOPK_NO_TOPP: skip both." }, "allow_advanced_sampling": { "default": false, "description": "DEPRECATED: no-op kept for backward compatibility. Will be removed in a future release. Non-greedy sampling is now auto-detected per request; this flag no longer has any effect.", "status": "deprecated", "title": "Allow Advanced Sampling", "type": "boolean" }, "decoding_type": { "const": "User_Provided", "default": "User_Provided", "title": "Decoding Type", "type": "string" }, "draft_len_schedule": { "anyOf": [ { "additionalProperties": { "type": "integer" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Developer interface: dynamically adjust draft length based on active batch size in runtime.Maps batch size to draft lengths.For example: draft_len_schedule = {4:4, 8:2, 32:1} - Batch sizes 1-4: use draft_len=4 - Batch sizes 5-8: use draft_len=2 - Batch sizes 9-32: use draft_len=1 - Batch sizes 33+: use draft_len=0 (implicit, speculation disabled). Mutually exclusive with max_concurrency since draft_len_schedule implicitly support max concurrency control.", "title": "Draft Len Schedule" }, "drafter": { "description": "A Drafter requires the Python LLM API.", "not": {} }, "enable_penalty": { "default": false, "description": "If true, enables the occurrence penalties (repetition / presence / frequency) for one-model speculative decoding. Off by default because the penalties need a [num_seq_slots, vocab_size] occurrence-count workspace that is allocated up front (CUDA graphs capture fixed buffer addresses). While off, a request that asks for any of these penalties is rejected at admission rather than silently decoded without them.", "status": "prototype", "title": "Enable Penalty", "type": "boolean" }, "load_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The load format of the speculative model.", "title": "Load Format" }, "max_concurrency": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "When specified (>0), speculation will be disabled at batch sizes above this value. Otherwise, speculation will always be on. PyTorch backend only. Mutually exclusive with max_concurrency since draft_len_schedule implicitly supports max concurrency control.", "title": "Max Concurrency" }, "max_draft_len": { "anyOf": [ { "minimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of draft tokens.", "title": "Max Draft Len" }, "max_total_draft_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of draft tokens in the draft tokens tree. If it's a linear tree, each draft layer will only generate one draft token. In this case, max_draft_len == max_total_draft_tokens. If it's a static or dynamic tree, each draft layer may generate more than one draft token. In this case, max_total_draft_tokens >= max_draft_len.", "title": "Max Total Draft Tokens" }, "moe_backend": { "anyOf": [ { "enum": [ "AUTO", "CUTLASS", "CUTEDSL", "CUTEDSL_FC12", "TRTLLM", "DEEPGEMM", "DENSEGEMM", "VANILLA", "TRITON", "MARLIN", "MEGAMOE_DEEPGEMM", "MEGAMOE_CUTEDSL" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "MoE backend override for a neural draft model or embedded MTP layers on the PyTorch backend. None inherits the target model's backend. AUTO resolves from the draft checkpoint or embedded MTP layer quantization, and a concrete backend applies only to the draft model or layers. Resolution may fall back based on model, quantization, and hardware support. Replacement-head MTP checkpoints must inherit the target model's MoE backend; leave this option unset. Nemotron-H embedded MTP layers must inherit the target backend because their checkpoint mapper uses a shared backend-dependent layout. Decoding methods without a neural draft model ignore this option.", "title": "Moe Backend" }, "resource_manager": { "type": "null" }, "speculative_model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "speculative_model_dir": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "type": "null" } ], "default": null, "description": "The speculative (draft) model. Accepts either (1) a HuggingFace Hub model ID (e.g. 'yuhuili/EAGLE3-LLaMA3.1-Instruct-8B'), which will be automatically downloaded, or (2) a local filesystem path to a downloaded model directory. For one-model MTP, a non-target checkpoint provides either replacement MTP heads or a complete external draft model, depending on the target model implementation. Pointing it at the target checkpoint uses the target's embedded mtp.* weights.", "title": "Speculative Model" }, "use_rejection_sampling": { "default": false, "description": "If true, enables rejection sampling for one-model speculative decoding paths when the batch contains any non-greedy request. All-greedy batches always take the argmax fast path regardless of this flag. Set to false (default) to use exact-match verification on non-greedy batches. The non-dynamic-tree one-model path requires FlashInfer.", "status": "prototype", "title": "Use Rejection Sampling", "type": "boolean" } }, "required": [ "drafter" ], "title": "UserProvidedDecodingConfig", "type": "object" }, "WaitingQueuePolicy": { "description": "Waiting queue scheduling policy for managing pending requests.", "enum": [ "fcfs", "priority" ], "title": "WaitingQueuePolicy", "type": "string" } }, "$id": "https://nvidia.github.io/TensorRT-LLM/1.4.0rc1/_static/schemas/trtllm-serve-disagg-config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "description": "Static YAML checks only; runtime validation remains authoritative.", "properties": { "_mpi_session": { "description": "Live objects require the Python LLM API.", "type": "null" }, "allow_request_chat_template": { "default": false, "title": "Allow Request Chat Template", "type": "boolean" }, "allreduce_strategy": { "anyOf": [ { "enum": [ "AUTO", "NCCL", "UB", "MINLATENCY", "ONESHOT", "TWOSHOT", "LOWPRECISION", "MNNVL", "NCCL_SYMMETRIC" ], "type": "string" }, { "type": "null" } ], "default": "AUTO", "description": "Allreduce strategy to use.", "status": "beta", "title": "Allreduce Strategy" }, "attention_dp_config": { "anyOf": [ { "$ref": "#/$defs/AttentionDpConfig" }, { "type": "null" } ], "default": null, "description": "Optimized load-balancing for the DP Attention scheduler.", "status": "beta" }, "attn_backend": { "default": "TRTLLM", "description": "Attention backend to use.", "status": "beta", "telemetry": { "allowed_values": [ "VANILLA", "TRTLLM", "FLASHINFER" ] }, "title": "Attn Backend", "type": "string" }, "backend": { "const": "pytorch", "default": "pytorch", "description": "The backend to use for this LLM instance.", "status": "deprecated", "title": "Backend", "type": "string" }, "batch_wait_max_tokens_ratio": { "default": 0, "description": "Token accumulation threshold ratio for batch scheduling optimization. If greater than 0, the scheduler will accumulate requests locally until the total token count reaches batch_wait_max_tokens_ratio * max_num_tokens. This mechanism enhances GPU utilization efficiency by ensuring adequate batch sizes. If 0, disables token-based batching delays.", "maximum": 1, "minimum": 0, "status": "prototype", "title": "Batch Wait Max Tokens Ratio", "type": "number" }, "batch_wait_timeout_iters": { "default": 0, "description": "Maximum number of iterations the scheduler will wait to accumulate new coming requests for improved GPU utilization efficiency. If greater than 0, the scheduler will delay batch processing to gather more requests up to the specified iteration limit. If 0, disables timeout-iters-based batching delays.", "minimum": 0, "status": "prototype", "title": "Batch Wait Timeout Iters", "type": "integer" }, "batch_wait_timeout_ms": { "default": 0, "description": "If greater than 0, the request queue might wait up to batch_wait_timeout_ms to receive max_batch_size requests, if fewer than max_batch_size requests are currently available. If 0, no waiting occurs.", "minimum": 0, "status": "prototype", "title": "Batch Wait Timeout Ms", "type": "number" }, "batched_logits_processor": { "description": "Live objects require the Python LLM API.", "type": "null" }, "bounded_replay_on_generation": { "default": false, "title": "Bounded Replay On Generation", "type": "boolean" }, "cache_transceiver_config": { "anyOf": [ { "$ref": "#/$defs/CacheTransceiverConfig" }, { "type": "null" } ], "default": null, "description": "Cache transceiver config.", "status": "prototype" }, "checkpoint_format": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The registered checkpoint loader format to use. `MX` selects ModelExpress as an opportunistic P2P loading path and falls back to loading the provided Hugging Face checkpoint; it does not require converting that checkpoint to an MX-specific format. You may use a custom checkpoint format by subclassing `BaseCheckpointLoader` and registering it with `register_checkpoint_loader`.\nIf neither checkpoint_format nor checkpoint_loader are provided, checkpoint_format will be set to HF and the default HfCheckpointLoader will be used.\nIf checkpoint_format and checkpoint_loader are both provided, checkpoint_loader will be ignored.", "status": "prototype", "title": "Checkpoint Format" }, "checkpoint_io_policy": { "default": "auto", "description": "Controls checkpoint storage I/O independently of checkpoint format. 'auto' selects rank-striped read-ahead for compatible built-in PyTorch/HF loads and selects native I/O otherwise. 'native' preserves the existing loader. 'rank_striped_read_ahead' lets node-local ranks read disjoint SafeTensors extents while native mapping, materialization, and H2D continue. Incompatible configurations select native I/O before optimized reader or collective setup. Runtime-ineligible loads fall back to native before model mutation.", "enum": [ "auto", "native", "rank_striped_read_ahead" ], "status": "prototype", "title": "Checkpoint Io Policy", "type": "string" }, "checkpoint_loader": { "description": "Live objects require the Python LLM API.", "type": "null" }, "conditional_disagg_config": { "anyOf": [ { "additionalProperties": false, "properties": { "max_local_prefill_length": { "default": 0, "title": "Max Local Prefill Length", "type": "integer" } }, "title": "ConditionalDisaggConfig", "type": "object" }, { "type": "null" } ] }, "context_parallel_size": { "default": 1, "description": "The context parallel size.", "title": "Context Parallel Size", "type": "integer" }, "context_servers": { "anyOf": [ { "$ref": "#/$defs/DisaggServerBlock" }, { "type": "null" } ] }, "conversation_affinity_header_for_subagents": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Conversation Affinity Header For Subagents" }, "cp_config": { "anyOf": [ { "$ref": "#/$defs/CpConfig" }, { "type": "null" } ], "default": null, "description": "Context parallel config.", "status": "prototype" }, "cuda_graph_config": { "anyOf": [ { "$ref": "#/$defs/DecodeCudaGraphConfig" }, { "$ref": "#/$defs/EncodeCudaGraphConfig" }, { "type": "null" } ], "description": "CUDA graph config. If true, use CUDA graphs for decoding. CUDA graphs are only created for the batch sizes in cuda_graph_config.batch_sizes, and are enabled for batches that consist of decoding requests *only* (the reason is that it's hard to capture a single graph with prefill requests since the input shapes are a function of the sequence lengths). Note that each CUDA graph can use up to 200 MB of extra memory.", "status": "beta", "title": "Cuda Graph Config" }, "custom_tokenizer": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Specify a custom tokenizer implementation. Accepts either: (1) a built-in alias (e.g., 'deepseek_v32'), or (2) a Python import path (e.g., 'tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer'). The tokenizer class must implement 'from_pretrained(path, **kwargs)' and the TokenizerBase interface.", "status": "prototype", "title": "Custom Tokenizer" }, "decoding_config": { "type": [ "object", "null" ] }, "disable_mm_encoder": { "default": false, "description": "Skip instantiating and loading the multimodal (e.g. vision) encoder of a multimodal checkpoint and serve it text-only. Saves the encoder's GPU memory (enlarging the KV cache pool) for workloads that never send image/video/audio inputs; such requests are rejected. Only takes effect for model implementations that support it (currently Mistral3 and the Qwen3-VL / Qwen3.5-VL family); a no-op otherwise. Defaults to False.", "status": "prototype", "title": "Disable Mm Encoder", "type": "boolean" }, "disable_overlap_scheduler": { "default": false, "description": "Disable the overlap scheduler.", "status": "beta", "title": "Disable Overlap Scheduler", "type": "boolean" }, "disagg_cluster": { "anyOf": [ { "additionalProperties": false, "properties": { "cluster_name": { "default": "", "title": "Cluster Name", "type": "string" }, "cluster_uri": { "title": "Cluster Uri", "type": "string" }, "heartbeat_interval_sec": { "default": 5, "title": "Heartbeat Interval Sec", "type": "integer" }, "inactive_timeout_sec": { "default": 10, "title": "Inactive Timeout Sec", "type": "integer" }, "minimal_instances": { "anyOf": [ { "$ref": "#/$defs/MinimalInstances" }, { "type": "null" } ], "default": null } }, "required": [ "cluster_uri" ], "title": "DisaggClusterConfig", "type": "object" }, { "type": "null" } ] }, "disagg_coordinator_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Disagg Coordinator Url" }, "dtype": { "default": "auto", "description": "The data type to use for the model. When 'auto' (default), it is read from the HF config.json ('dtype', or the deprecated 'torch_dtype'); for composite/VLM configs it falls back to the nested text_config.dtype. Defaults to bfloat16 if none is found.", "telemetry": { "allowed_values": [ "auto", "float16", "bfloat16", "float32" ] }, "title": "Dtype", "type": "string" }, "dwdp_config": { "anyOf": [ { "$ref": "#/$defs/DwdpConfig" }, { "type": "null" } ], "default": null, "description": "DWDP (Distributed Weight Data Parallelism) config.", "status": "prototype" }, "enable_attention_dp": { "default": false, "description": "Enable attention data parallel.", "status": "beta", "title": "Enable Attention Dp", "type": "boolean" }, "enable_autotuner": { "default": true, "description": "Enable autotuner for all tunable ops. This flag is for debugging purposes only, and the performance may significantly degrade if set to false.", "status": "prototype", "title": "Enable Autotuner", "type": "boolean" }, "enable_chunked_prefill": { "default": false, "description": "Enable chunked prefill.", "title": "Enable Chunked Prefill", "type": "boolean" }, "enable_early_first_token_response": { "default": false, "description": "Under the overlap scheduler, emit the first-token response ahead of the next sample step to reduce TTFT. No effect when the overlap scheduler is disabled.", "status": "prototype", "title": "Enable Early First Token Response", "type": "boolean" }, "enable_encoder_decoder_mixed_cuda_graph": { "default": true, "description": "Enable the mixed-batch CUDA graph performance optimization for encoder-decoder models. The graph handles decoder iterations containing both context and generation requests. It is enabled by default when both `cuda_graph_config` and `encoder_cuda_graph_config` produce usable graph shapes.", "status": "prototype", "title": "Enable Encoder Decoder Mixed Cuda Graph", "type": "boolean" }, "enable_energy_metrics": { "default": false, "description": "Enable GPU energy monitoring via NVML. When enabled, the server exposes an /energy_metrics endpoint that reports cumulative GPU energy consumption in joules.", "status": "prototype", "title": "Enable Energy Metrics", "type": "boolean" }, "enable_in_graph_sampling": { "default": false, "description": "Opt-in in-graph sampling: capture part of the sampling work into the model's CUDA graph and run it there, instead of eagerly after the forward. The sampler itself is unchanged; only where its work runs differs. Batches whose sampling cannot be captured are unaffected and keep the eager path. This captures an additional set of CUDA graphs at startup, which makes warmup noticeably slower and costs extra memory, in exchange for lower per-step sampling overhead.", "status": "prototype", "title": "Enable In Graph Sampling", "type": "boolean" }, "enable_iter_perf_stats": { "default": false, "description": "Enable iteration performance statistics.", "status": "prototype", "title": "Enable Iter Perf Stats", "type": "boolean" }, "enable_iter_req_stats": { "default": false, "description": "If true, enables per request stats per iteration. Must also set enable_iter_perf_stats to true to get request stats.", "status": "prototype", "title": "Enable Iter Req Stats", "type": "boolean" }, "enable_layerwise_nvtx_marker": { "default": false, "description": "If true, enable layerwise nvtx marker.", "status": "beta", "title": "Enable Layerwise Nvtx Marker", "type": "boolean" }, "enable_lm_head_tp_in_adp": { "default": false, "description": "Enable LM head TP in attention dp.", "status": "prototype", "title": "Enable Lm Head Tp In Adp", "type": "boolean" }, "enable_locality_domains": { "default": false, "description": "Enable locality domain execution for supported PyTorch backend ops.", "status": "prototype", "title": "Enable Locality Domains", "type": "boolean" }, "enable_lora": { "default": false, "description": "Enable LoRA.", "title": "Enable Lora", "type": "boolean" }, "enable_low_latency_host_dispatch": { "default": false, "description": "Use low-latency spin-wait mode for CUDA host task dispatch (cudaLaunchHostFunc_v2 with cudaHostTaskSpinWait). Reduces callback latency at the cost of a CPU core spinning while waiting for the GPU event. Requires CUDA 13.2+; on older CUDA versions, falls back to the default blocking mode and logs a one-time warning.", "status": "prototype", "title": "Enable Low Latency Host Dispatch", "type": "boolean" }, "enable_min_latency": { "default": false, "description": "If true, enable min-latency mode. Currently only used for Llama4.", "status": "beta", "title": "Enable Min Latency", "type": "boolean" }, "enable_mla_skip_correction": { "default": false, "description": "Enable threshold-based skip-correction for trtllm-gen MLA attention kernels on SM100 and SM103. When enabled, mla_skip_correction_threshold controls the optimization threshold.", "status": "prototype", "title": "Enable Mla Skip Correction", "type": "boolean" }, "enable_resource_governor": { "default": false, "description": "Enable the resource governor for runtime cache management operations such as KV cache truncation. This adds a per-iteration broadcast collective.", "status": "prototype", "title": "Enable Resource Governor", "type": "boolean" }, "enable_return_routed_experts": { "default": false, "description": "Router Replay (R3): capture per-token pre-EPLB logical top-k MoE expert ids so they can be returned on outputs (per request via SamplingParams.return_routed_experts), for train/inference routing alignment in MoE reinforcement learning. Zero overhead when disabled. Separated-routing MoE backends only.", "status": "beta", "title": "Enable Return Routed Experts", "type": "boolean" }, "enable_speculative_beam_history_d2h": { "default": false, "description": "Opt-in beam-search optimization: skip per-step beam-history D2H copies on likely-non-terminal steps via a host-side predictor and route the remaining copies through a private side stream. Mispredictions fall back to a synchronous .cpu(), preserving correctness but breaking overlap on that step. Incompatible with the async D2H worker (sampler_force_async_worker=True or confidential compute).", "status": "prototype", "title": "Enable Speculative Beam History D2H", "type": "boolean" }, "enable_tokenization_cache": { "default": false, "description": "Cache the tokenization of recent prompts so that a prompt extending a cached one only tokenizes its tail, which speeds up multi-turn serving with long prompts. Requires a fast tokenizer and applies only to prompts tokenized with add_special_tokens=False and no truncation. The output is identical to tokenizing the whole prompt.", "status": "prototype", "title": "Enable Tokenization Cache", "type": "boolean" }, "encode_only": { "default": false, "description": "Set to True to use the batch-forward encode() path, which runs a single forward pass and returns the model output directly, bypassing the scheduler and autoregressive loop. Works for encoder-only models (BERT, RoBERTa, reward models) and decoder models used in single-prefill mode (e.g., extracting embeddings). When False (default), uses the standard generate() path.", "status": "prototype", "title": "Encode Only", "type": "boolean" }, "encoder_cuda_graph_config": { "anyOf": [ { "$ref": "#/$defs/EncodeCudaGraphConfig" }, { "type": "null" } ], "default": null, "description": "CUDA graph configuration for the encoder forward pass of an encoder-decoder model. Use `cuda_graph_config` for the decoder and this field for the encoder. Encoder CUDA graphs require `encoder_max_batch_size` to be set.", "status": "prototype" }, "encoder_max_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of top-level encoder inputs processed in one iteration. For encoder-decoder models, each encoder request counts as one input. For multimodal models, each atomic image, video, or other encoder item counts as one input, even if it expands into multiple internal attention sequences. For encoder-decoder models, it also limits encoder CUDA graph batch sizes. Falls back to `max_batch_size` when unset.", "status": "prototype", "title": "Encoder Max Batch Size" }, "encoder_max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Maximum number of encoder tokens. For encoder-decoder models, this limits encoder CUDA graph total-token buckets. For multimodal models, it limits encoder attention tokens scheduled in one iteration and is shared across all encoded modalities. It falls back to `max_num_tokens` when unset. Because an atomic multimodal item cannot be split, the effective budget is raised to the model's largest atomic item when necessary.", "status": "prototype", "title": "Encoder Max Num Tokens" }, "env_overrides": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won't update unless the code fetches them from os.environ on demand.", "status": "prototype", "title": "Env Overrides" }, "force_dynamic_quantization": { "default": false, "description": "If true, force dynamic quantization. Defaults to False.", "status": "prototype", "title": "Force Dynamic Quantization", "type": "boolean" }, "free_gpu_memory_fraction": { "anyOf": [ { "maximum": 1, "minimum": 0, "type": "number" }, { "type": "null" } ], "default": 0.9, "description": "The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.", "title": "Free Gpu Memory Fraction" }, "garbage_collection_gen0_threshold": { "default": 20000, "description": "Threshold for Python garbage collection of generation 0 objects. Lower values trigger more frequent garbage collection.", "status": "beta", "title": "Garbage Collection Gen0 Threshold", "type": "integer" }, "gather_generation_logits": { "default": false, "description": "Gather generation logits.", "status": "prototype", "title": "Gather Generation Logits", "type": "boolean" }, "gen_strip_message_history": { "default": false, "title": "Gen Strip Message History", "type": "boolean" }, "gen_tokids_ctxbytes": { "default": false, "title": "Gen Tokids Ctxbytes", "type": "boolean" }, "generation_config": { "default": "trtllm", "description": "Controls whether sampling defaults are loaded from the model's generation_config.json. 'auto' applies supported values when the request does not specify them; 'trtllm' preserves TRT-LLM defaults. Precedence is request values, generation_config.json values, then TRT-LLM defaults.", "enum": [ "auto", "trtllm" ], "status": "prototype", "title": "Generation Config", "type": "string" }, "generation_servers": { "anyOf": [ { "$ref": "#/$defs/DisaggServerBlock" }, { "type": "null" } ] }, "gms_config": { "$ref": "#/$defs/GmsConfig", "description": "GPU Memory Service (GMS) weight sharing config.", "status": "prototype" }, "gpus_per_node": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The number of GPUs per node.", "status": "beta", "title": "Gpus Per Node" }, "guided_decoding_backend": { "anyOf": [ { "enum": [ "xgrammar", "llguidance" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Guided decoding backend. llguidance is supported in PyTorch backend only.", "title": "Guided Decoding Backend" }, "hf_revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the model.", "title": "Revision" }, "hostname": { "default": "localhost", "title": "Hostname", "type": "string" }, "internal_request_auth_key": { "anyOf": [ { "minLength": 1, "type": "string" }, { "type": "null" } ] }, "iter_perf_stats_interval": { "default": 1, "description": "Build an iteration statistics record only every N executor iterations when enable_iter_perf_stats is true, which reduces the host overhead of collecting the statistics (including the per-request statistics of enable_iter_req_stats). A value of 1 builds a record every iteration. With N > 1, numCompletedRequests, numNewActiveRequests and the KV cache iteration deltas of skipped iterations are carried into a later record, so their totals stay exact but can be reported late. Without attention DP, one extra record is emitted when the last active request finishes in a skipped iteration; with attention DP, what is left after the last record of a busy period is reported when the executor resumes. All other fields describe only the sampled iteration, so Prometheus counters built from them (e.g. speculative decoding draft and accepted token totals) reach only about 1/N of the true totals, and gauges such as KV cache utilization refresh every N iterations. With enable_iter_req_stats, requests that finish in a skipped iteration get no requestStats entry.", "exclusiveMinimum": 0, "status": "prototype", "title": "Iter Perf Stats Interval", "type": "integer" }, "iter_stats_max_iterations": { "anyOf": [ { "minimum": -1, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of iterations for iter stats. Set to -1 to keep all iteration stats. Set to 0 to disable iteration stats in the TensorRT executor.", "status": "prototype", "title": "Iter Stats Max Iterations" }, "kv_cache_compression_config": { "anyOf": [ { "discriminator": { "mapping": { "quantization_for_cold_page": "#/$defs/ColdPageQuantizationCompressionConfig", "triattention": "#/$defs/TriAttentionKvCacheCompressionConfig" }, "propertyName": "algorithm" }, "oneOf": [ { "$ref": "#/$defs/ColdPageQuantizationCompressionConfig" }, { "$ref": "#/$defs/TriAttentionKvCacheCompressionConfig" } ] }, { "type": "null" } ], "default": null, "description": "KV-cache compression config; None disables compression.", "status": "prototype", "title": "Kv Cache Compression Config" }, "kv_cache_config": { "$ref": "#/$defs/KvCacheConfig", "description": "KV cache config." }, "kv_cache_dtype": { "default": "auto", "description": "The data type for the KV cache. 'auto' (default) leaves the checkpoint's own KV-cache quantization metadata untouched (quant_config.kv_cache_quant_algo is inherited as-is); 'fp8', 'fp8_ds_mla', or 'nvfp4' override it explicitly. 'fp8_ds_mla' selects the packed FP8 cache used by sparse MLA on SM90/SM120/SM121. Resolved at LLM-construction time, including when set via trtllm-serve --extra_llm_api_options.", "telemetry": { "allowed_values": [ "auto", "float16", "bfloat16", "float32", "fp8", "fp8_ds_mla", "nvfp4" ] }, "title": "Dtype", "type": "string" }, "kv_connector_config": { "anyOf": [ { "$ref": "#/$defs/KvCacheConnectorConfig" }, { "type": "null" } ], "default": null, "description": "The config for KV cache connector.", "status": "prototype" }, "layer_wise_benchmarks_config": { "$ref": "#/$defs/LayerwiseBenchmarksConfig", "status": "prototype" }, "load_format": { "anyOf": [ { "type": "string" }, { "$ref": "#/$defs/LoadFormat" } ], "default": 0, "description": "How to load the model weights. By default, detect the weight type from the model checkpoint.", "telemetry": { "allowed_values": [ "auto", "dummy", "vision_only", "gms" ] }, "title": "Load Format" }, "lora_config": { "anyOf": [ { "$ref": "#/$defs/LoraConfig" }, { "type": "null" } ], "default": null, "description": "LoRA configuration for the model." }, "max_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 2048, "description": "The maximum batch size.", "title": "Max Batch Size" }, "max_beam_width": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 1, "description": "The maximum beam width.", "title": "Max Beam Width" }, "max_input_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 1024, "description": "The maximum input length.", "title": "Max Input Len" }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 8192, "description": "The maximum number of tokens.", "title": "Max Num Tokens" }, "max_retries": { "default": 1, "title": "Max Retries", "type": "integer" }, "max_seq_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum sequence length.", "title": "Max Seq Len" }, "max_stats_len": { "default": 1000, "description": "The max number of performance statistic entries. Set to -1 to keep all entries. Set to 0 to use a minimum buffer size of 1.", "minimum": -1, "status": "prototype", "title": "Max Stats Len", "type": "integer" }, "mla_skip_correction_threshold": { "default": 8.0, "description": "Threshold for threshold-based skip-correction. This is used only when enable_mla_skip_correction is True. The default is 8. The maximum supported value depends on the selected kernel's BMM2 dtype: 8 for E4M3, 15 for FP16, and 32 for BF16.", "exclusiveMinimum": 0.0, "maximum": 32.0, "status": "prototype", "title": "Mla Skip Correction Threshold", "type": "number" }, "mm_encoder_only": { "default": false, "description": "Only load/execute the vision encoder part of the full model. Defaults to False.", "status": "prototype", "title": "Mm Encoder Only", "type": "boolean" }, "model": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" } ], "description": "The path to the model checkpoint or the model name from the Hugging Face Hub.", "title": "Model" }, "model_kwargs": { "anyOf": [ { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "Optional parameters overriding model config defaults. Precedence: (1) model_kwargs, (2) model config file, (3) model config class defaults. Unknown keys are ignored", "status": "prototype", "title": "Model Kwargs" }, "modeling_v2": { "default": "off", "description": "Whether to build the model from a modeling_v2 target (experimental) instead of the built-in implementation. 'off' never does. 'auto' uses a target when one claims this deployment and falls back to the built-in implementation otherwise. 'require' raises instead of falling back.", "enum": [ "off", "auto", "require" ], "status": "prototype", "title": "Modeling V2", "type": "string" }, "moe_cluster_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The cluster parallel size for MoE model's expert weights.", "status": "deprecated", "title": "Moe Cluster Parallel Size" }, "moe_config": { "$ref": "#/$defs/MoeConfig", "description": "MoE config.", "status": "beta" }, "moe_expert_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The expert parallel size for MoE model's expert weights.", "title": "Moe Expert Parallel Size" }, "moe_tensor_parallel_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "The tensor parallel size for MoE model's expert weights.", "title": "Moe Tensor Parallel Size" }, "multimodal_config": { "anyOf": [ { "$ref": "#/$defs/MultimodalConfig" }, { "type": "null" } ], "description": "Multimodal model configuration.", "status": "prototype" }, "mx_config": { "$ref": "#/$defs/ModelExpressConfig", "description": "ModelExpress (MX) P2P checkpoint loading config.", "status": "prototype" }, "node_id": { "maximum": 255, "minimum": 0, "type": [ "integer", "null" ] }, "num_postprocess_workers": { "default": 0, "description": "The number of processes used for postprocessing the generated tokens, including detokenization.", "status": "prototype", "title": "Num Postprocess Workers", "type": "integer" }, "num_serve_frontends": { "default": 1, "description": "The number of HTTP or OpenEngine frontend processes serving one executor. Used by trtllm-serve: values > 1 run additional attached frontend processes that share the serving port via SO_REUSEPORT (classic IPC executor path only).", "maximum": 64, "minimum": 1, "status": "prototype", "title": "Num Serve Frontends", "type": "integer" }, "num_workers": { "default": 1, "title": "Num Workers", "type": "integer" }, "nvfp4_gemm_config": { "$ref": "#/$defs/Nvfp4GemmConfig", "description": "NVFP4 GEMM backend config.", "status": "beta" }, "orchestrator_type": { "anyOf": [ { "enum": [ "rpc", "ray" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "The orchestrator type to use. Defaults to None, which uses MPI.", "status": "prototype", "title": "Orchestrator Type" }, "otlp_config": { "anyOf": [ { "additionalProperties": false, "properties": { "otlp_traces_endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Otlp Traces Endpoint" } }, "title": "OtlpConfig", "type": "object" }, { "type": "null" } ] }, "otlp_traces_endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Target URL to which OpenTelemetry traces will be sent.", "status": "prototype", "title": "Otlp Traces Endpoint" }, "peft_cache_config": { "anyOf": [ { "$ref": "#/$defs/PeftCacheConfig" }, { "type": "null" } ], "default": null, "description": "PEFT cache config.", "status": "prototype" }, "per_request_spec_decode_stats": { "default": false, "description": "Include per-request speculative-decoding acceptance statistics on each response choice. Server-side opt-in only: unlike return_perf_metrics this needs no per-request header, so benchmarking clients that discover the payload by shape do not have to know they are talking to TensorRT-LLM. Deliberately independent of return_perf_metrics, which also mounts the Prometheus endpoint. PyTorch backend only.", "status": "prototype", "title": "Per Request Spec Decode Stats", "type": "boolean" }, "perf_metrics_max_requests": { "default": 0, "title": "Perf Metrics Max Requests", "type": "integer" }, "perf_metrics_output_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Perf Metrics Output Dir" }, "pipeline_parallel_size": { "default": 1, "description": "The pipeline parallel size.", "title": "Pipeline Parallel Size", "type": "integer" }, "port": { "default": 8000, "title": "Port", "type": "integer" }, "post_processor_hook": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Python import path of a user post-processing hook applied after detokenization and before the per-endpoint response formatter (e.g. 'my_pkg.guardrail.MyPostProcessorHook'). The class must be importable and picklable, take no constructor arguments, and be callable as '__call__(chunk) -> verdict' (see tensorrt_llm.executor.postprocessor_hook). It runs once per output, per streaming chunk, and may rewrite, suppress, or terminate the output; it owns its own per-request state.", "status": "prototype", "title": "Post Processor Hook" }, "postprocess_tokenizer_dir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The path to the tokenizer directory for postprocessing.", "status": "prototype", "title": "Postprocess Tokenizer Dir" }, "pp_partition": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Pipeline parallel partition, a list of each rank's layer number.", "status": "prototype", "title": "Pp Partition" }, "prefill_capture_num_tokens": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Token-count buckets captured by the selected prefill CUDA graph implementation.", "status": "prototype", "title": "Prefill Capture Num Tokens" }, "prefill_cuda_graph_backend": { "$ref": "#/$defs/PrefillCudaGraphBackend", "default": "disabled", "description": "CUDA graph implementation used for prefill requests. Defaults to disabled.", "status": "prototype" }, "print_iter_log": { "default": false, "description": "Print iteration logs.", "status": "beta", "title": "Print Iter Log", "type": "boolean" }, "prometheus_metrics_config": { "anyOf": [ { "$ref": "#/$defs/PrometheusMetricsConfig" }, { "type": "null" } ], "default": null, "description": "Configuration for Prometheus metrics collection, including custom histogram bucket boundaries.", "status": "prototype" }, "ray_placement_config": { "anyOf": [ { "$ref": "#/$defs/RayPlacementConfig" }, { "type": "null" } ], "default": null, "description": "Placement config for RayGPUWorker. Only used with AsyncLLM and orchestrator_type='ray'.", "status": "prototype" }, "ray_worker_extension_cls": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The full worker extension class name including module path. Allows users to extend the functions of the RayGPUWorker class.", "status": "prototype", "title": "Ray Worker Extension Cls" }, "ray_worker_nsight_options": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "default": null, "description": "Nsight options.", "status": "prototype", "title": "Ray Worker Nsight Options" }, "reasoning_parser": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The parser to separate reasoning content from output.", "status": "prototype", "telemetry": { "allowed_values": [ "auto", "deepseek-r1", "poolside_v1", "laguna", "qwen3", "qwen3_5", "minimax_m2", "minimax_m2_append_think", "nano-v3", "gemma4", "kimi_k2", "kimi_k25", "k-exaone" ] }, "title": "Reasoning Parser" }, "reorder_policy_config": { "anyOf": [ { "$ref": "#/$defs/ReorderRequestPolicyConfig" }, { "type": "null" } ], "default": null, "description": "The request reordering policy to use.", "status": "prototype" }, "request_stats_max_iterations": { "anyOf": [ { "minimum": -1, "type": "integer" }, { "type": "null" } ], "default": null, "description": "The maximum number of iterations for request stats. Set to -1 to keep all request stats. Set to 0 to disable request stats.", "status": "prototype", "title": "Request Stats Max Iterations" }, "return_perf_metrics": { "default": false, "title": "Return Perf Metrics", "type": "boolean" }, "revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the model.", "title": "Revision" }, "sampler_force_async_worker": { "default": false, "description": "Force usage of the async worker in the sampler for D2H copies, even if confidential compute is not active. Normally, the async worker should only be used when confidential compute is active. This argument is provided to enable it for testing purposes, irrespective of confidential compute state.", "status": "prototype", "title": "Sampler Force Async Worker", "type": "boolean" }, "schedule_style": { "default": "context_first", "enum": [ "context_first", "generation_first" ], "title": "Schedule Style", "type": "string" }, "scheduler_config": { "$ref": "#/$defs/SchedulerConfig", "description": "Scheduler config.", "status": "prototype" }, "server_keep_alive_timeout": { "default": 10, "title": "Server Keep Alive Timeout", "type": "integer" }, "skip_tokenizer_init": { "default": false, "description": "Whether to skip the tokenizer initialization.", "title": "Skip Tokenizer Init", "type": "boolean" }, "sleep_config": { "anyOf": [ { "$ref": "#/$defs/SleepConfig" }, { "type": "null" } ], "default": null, "description": "Configuration for the LLM sleep feature. Sleep feature requires extra setup that may slow down model loading. Only enable it if you intend to use this feature.", "status": "prototype" }, "sparse_attention_config": { "anyOf": [ { "discriminator": { "mapping": { "csa2": "#/$defs/CSA2SparseAttentionConfig", "deepseek_v4": "#/$defs/DeepSeekV4SparseAttentionConfig", "dsa": "#/$defs/DeepSeekSparseAttentionConfig", "minimax_m3": "#/$defs/MiniMaxM3SparseAttentionConfig", "qsa": "#/$defs/QSASparseAttentionConfig", "rocket": "#/$defs/RocketSparseAttentionConfig", "skip_softmax": "#/$defs/SkipSoftmaxAttentionConfig" }, "propertyName": "algorithm" }, "oneOf": [ { "$ref": "#/$defs/QSASparseAttentionConfig" }, { "$ref": "#/$defs/RocketSparseAttentionConfig" }, { "$ref": "#/$defs/DeepSeekSparseAttentionConfig" }, { "$ref": "#/$defs/DeepSeekV4SparseAttentionConfig" }, { "$ref": "#/$defs/CSA2SparseAttentionConfig" }, { "$ref": "#/$defs/SkipSoftmaxAttentionConfig" }, { "$ref": "#/$defs/MiniMaxM3SparseAttentionConfig" } ] }, { "type": "null" } ], "default": null, "description": "Sparse attention config.", "status": "prototype", "title": "Sparse Attention Config" }, "speculative_config": { "anyOf": [ { "discriminator": { "mapping": { "AUTO": "#/$defs/AutoDecodingConfig", "DFlash": "#/$defs/DFlashDecodingConfig", "DSpark": "#/$defs/DSparkDecodingConfig", "Draft_Target": "#/$defs/DraftTargetDecodingConfig", "Eagle": "#/$defs/EagleDecodingConfig", "Eagle3": "#/$defs/Eagle3DecodingConfig", "MTP": "#/$defs/MTPDecodingConfig", "NGram": "#/$defs/NGramDecodingConfig", "PARD": "#/$defs/PARDDecodingConfig", "SA": "#/$defs/SADecodingConfig", "SaveState": "#/$defs/SaveHiddenStatesDecodingConfig", "User_Provided": "#/$defs/UserProvidedDecodingConfig" }, "propertyName": "decoding_type" }, "oneOf": [ { "$ref": "#/$defs/DraftTargetDecodingConfig" }, { "$ref": "#/$defs/Eagle3DecodingConfig" }, { "$ref": "#/$defs/EagleDecodingConfig" }, { "$ref": "#/$defs/MTPDecodingConfig" }, { "$ref": "#/$defs/NGramDecodingConfig" }, { "$ref": "#/$defs/SADecodingConfig" }, { "$ref": "#/$defs/UserProvidedDecodingConfig" }, { "$ref": "#/$defs/SaveHiddenStatesDecodingConfig" }, { "$ref": "#/$defs/PARDDecodingConfig" }, { "$ref": "#/$defs/DFlashDecodingConfig" }, { "$ref": "#/$defs/DSparkDecodingConfig" }, { "$ref": "#/$defs/AutoDecodingConfig" } ] }, { "type": "null" } ], "default": null, "description": "Speculative decoding config.", "title": "Speculative Config" }, "stream_interval": { "default": 1, "description": "The iteration interval to create responses under the streaming mode. Set this to a larger value when the batch size is large, which helps reduce the streaming overhead.", "exclusiveMinimum": 0, "title": "Stream Interval", "type": "integer" }, "subagent_affinity_scope": { "default": "context", "enum": [ "context", "both" ], "title": "Subagent Affinity Scope", "type": "string" }, "telemetry_config": { "anyOf": [ { "$ref": "#/$defs/TelemetryConfig", "description": "Telemetry configuration (opt-out, usage context).", "status": "prototype" }, { "type": "null" } ] }, "tensor_parallel_size": { "default": 1, "description": "The tensor parallel size.", "title": "Tensor Parallel Size", "type": "integer" }, "tokenizer": { "anyOf": [ { "type": "string" }, { "format": "path", "type": "string" }, { "not": {} }, { "type": "null" } ], "default": null, "description": "The path to the tokenizer checkpoint or the tokenizer name from the Hugging Face Hub.", "title": "Tokenizer" }, "tokenizer_mode": { "default": "auto", "description": "The mode to initialize the tokenizer.", "enum": [ "auto", "slow" ], "title": "Tokenizer Mode", "type": "string" }, "tokenizer_revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "The revision to use for the tokenizer.", "title": "Tokenizer Revision" }, "torch_compile_config": { "anyOf": [ { "$ref": "#/$defs/TorchCompileConfig" }, { "type": "null" } ], "default": null, "description": "Torch compile config.", "status": "prototype" }, "trust_remote_code": { "default": false, "description": "Whether to trust the remote code.", "title": "Trust Remote Code", "type": "boolean" }, "use_cute_dsl_bf16_bmm": { "default": false, "description": "If true, use CuTe DSL BF16 BMM on Blackwell (SM100/SM103) and Rubin (SM107), including the DeepSeek-V4 o_a projection on Rubin when DSL support and dimension alignment permit. Defaults to false; automatically enabled with pipeline_parallel_size > 1 on SM100/SM103/SM107.", "status": "prototype", "title": "Use Cute Dsl Bf16 Bmm", "type": "boolean" }, "use_cute_dsl_bf16_gemm": { "default": false, "description": "If true, use CuTe DSL BF16 persistent GEMM for Linear layers on Blackwell (SM100/SM103) and Rubin (SM107) when supported. Defaults to false; automatically enabled with pipeline_parallel_size > 1 on SM100/SM103/SM107.", "status": "prototype", "title": "Use Cute Dsl Bf16 Gemm", "type": "boolean" }, "use_cute_dsl_blockscaling_bmm": { "default": false, "description": "If true, use CuTe DSL fp8 blockscaling bmm implementation.", "status": "prototype", "title": "Use Cute Dsl Blockscaling Bmm", "type": "boolean" }, "use_cute_dsl_blockscaling_mm": { "default": false, "description": "If true, use CuTe DSL fp8 blockscaling mm implementation.", "status": "prototype", "title": "Use Cute Dsl Blockscaling Mm", "type": "boolean" }, "use_fine_grained_sync": { "default": false, "description": "Enable fine-grained synchronization for MoE kernels on SM107. The FC1 producer kernel signals per-tile completion flags in device memory and the FC2 consumer kernel waits on them, so the two GEMMs overlap instead of serializing at kernel launch boundaries.", "status": "prototype", "title": "Use Fine Grained Sync", "type": "boolean" }, "video_pruning_rate": { "anyOf": [ { "exclusiveMaximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Pruning rate for video frames in multimodal models for Efficient Video Sampling (EVS). NOTE: this is currently only implemented in nemotron multimodal models. None (default) disables EVS, values in [0, 1) enable pruning.", "status": "prototype", "title": "Video Pruning Rate" } }, "required": [], "title": "TensorRT-LLM 1.4.0rc1 trtllm-serve disaggregated --config", "type": "object" }