{ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0. Generated by scripts/generate_trtllm_serve_schemas.py; do not edit by hand.", "$defs": { "AttentionConfig": { "additionalProperties": false, "description": "Configuration for Attention layers.", "properties": { "backend": { "default": "VANILLA", "description": "Attention backend: VANILLA (PyTorch SDPA), TRTLLM, CUDNN, FLASHINFER, CUTEDSL, FA4", "enum": [ "VANILLA", "TRTLLM", "CUDNN", "FLASHINFER", "CUTEDSL", "FA4" ], "status": "prototype", "title": "Backend", "type": "string" }, "quant_attention_config": { "anyOf": [ { "$ref": "#/$defs/QuantAttentionConfig" }, { "type": "null" } ], "default": null, "description": "Quantized-attention recipe (TRTLLM / CUDNN / FLASHINFER / CUTEDSL backends). Set to a QuantAttentionConfig instance to enable quantized attention; leave as None to disable.", "status": "prototype" }, "sparse_attention_config": { "anyOf": [ { "discriminator": { "mapping": { "skip_softmax": "#/$defs/SkipSoftmaxAttentionConfig", "sol_attn": "#/$defs/SolAttentionConfig", "vsa": "#/$defs/VideoSparseAttentionConfig" }, "propertyName": "algorithm" }, "oneOf": [ { "$ref": "#/$defs/SkipSoftmaxAttentionConfig" }, { "$ref": "#/$defs/VideoSparseAttentionConfig" }, { "$ref": "#/$defs/SolAttentionConfig" } ] }, { "type": "null" } ], "default": null, "description": "Sparse attention recipe. Discriminated by algorithm: skip_softmax (TRTLLM / CUTEDSL backends), vsa (CUTEDSL / TRTLLM backends), or sol_attn (TRTLLM / CUTEDSL backends).", "status": "prototype", "title": "Sparse Attention Config" } }, "title": "AttentionConfig", "type": "object" }, "CacheDiTConfig": { "additionalProperties": false, "description": "Configuration for Cache-DiT (DBCache, TaylorSeer, SCM).\n\nRequires the cache-dit package.", "properties": { "Bn_compute_blocks": { "default": 0, "description": "Last n blocks use residual cache (Bn).", "minimum": 0, "status": "prototype", "title": "Bn Compute Blocks", "type": "integer" }, "Fn_compute_blocks": { "default": 1, "description": "First n blocks always computed (Fn).", "minimum": 0, "status": "prototype", "title": "Fn Compute Blocks", "type": "integer" }, "cache_backend": { "const": "cache_dit", "default": "cache_dit", "title": "Cache Backend", "type": "string" }, "enable_separate_cfg": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "default": null, "description": "If set, forwarded to DBCacheConfig.enable_separate_cfg. If None, enablers pick defaults for each pipeline (Wan: batched CFG \u2192 False).", "status": "prototype", "title": "Enable Separate Cfg" }, "enable_taylorseer": { "default": false, "status": "prototype", "title": "Enable Taylorseer", "type": "boolean" }, "force_refresh_step_hint": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": null, "description": "Optional step index hint for forced cache refresh (cache_dit DBCacheConfig).", "status": "prototype", "title": "Force Refresh Step Hint" }, "force_refresh_step_policy": { "default": "once", "description": "Policy for force_refresh_step_hint: once or repeat.", "enum": [ "once", "repeat" ], "status": "prototype", "title": "Force Refresh Step Policy", "type": "string" }, "max_cached_steps": { "default": -1, "description": "Cap on cached steps; -1 means no limit.", "status": "prototype", "title": "Max Cached Steps", "type": "integer" }, "max_continuous_cached_steps": { "default": 3, "description": "Cap on consecutive cached steps (-1 = library unlimited; default 3).", "minimum": -1, "status": "prototype", "title": "Max Continuous Cached Steps", "type": "integer" }, "max_warmup_steps": { "default": 4, "description": "Initial steps that do not use cache (default tuned for few-step runs).", "minimum": 0, "status": "prototype", "title": "Max Warmup Steps", "type": "integer" }, "residual_diff_threshold": { "default": 0.24, "description": "L1 diff threshold for DBCache (default pairs with max_continuous_cached_steps).", "minimum": 0.0, "status": "prototype", "title": "Residual Diff Threshold", "type": "number" }, "scm_steps_mask_policy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Policy name for cache_dit.steps_mask (e.g. fast, medium, slow, ultra).", "status": "prototype", "title": "Scm Steps Mask Policy" }, "scm_steps_policy": { "default": "dynamic", "enum": [ "dynamic", "static" ], "status": "prototype", "title": "Scm Steps Policy", "type": "string" }, "taylorseer_order": { "default": 1, "maximum": 4, "minimum": 1, "status": "prototype", "title": "Taylorseer Order", "type": "integer" } }, "title": "CacheDiTConfig", "type": "object" }, "CompilationConfig": { "additionalProperties": false, "description": "Configuration for torch.compile / CUDA graph warmup shapes.\n\nWarmup shapes are the Cartesian product of ``resolutions`` and\n``num_frames``. More shapes mean slower startup but lower recompile\nrisk on first requests; fewer shapes mean faster startup but a recompile\non the first request with an un-warmed shape. When the fields are\n``None``, each model pipeline supplies its own defaults\n(e.g. Wan uses ``[(480, 832), (720, 1280)] x [33, 81]``).", "properties": { "num_frames": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of frame counts to warmup at startup. Combined with resolutions via Cartesian product. If None, uses model-specific defaults. For image models, use [1].", "status": "prototype", "title": "Num Frames" }, "resolutions": { "anyOf": [ { "items": { "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "integer" }, { "type": "integer" } ], "type": "array" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "List of (height, width) resolutions to warmup at startup. Combined with num_frames via Cartesian product. If None, uses model-specific defaults.", "status": "prototype", "title": "Resolutions" }, "skip_warmup": { "default": false, "description": "Skip the post-load warmup pass (compile + capture).", "status": "prototype", "title": "Skip Warmup", "type": "boolean" } }, "title": "CompilationConfig", "type": "object" }, "CpuOffloadConfig": { "additionalProperties": false, "description": "Configuration for offloading visual-generation model components to CPU.\n\nA diffusion pipeline is a sequence of large *components* (text encoder,\ndenoising transformer, VAE, optional guardrails, ...) that run one after\nanother, so only one is needed on the GPU at any moment. Offloading\nexploits this: weights are loaded and quantized normally, then held in\npacked host (CPU) storage and brought onto the GPU one *stage* at a time,\nrebinding each module's parameters/buffers to a reusable GPU arena while\nthat stage runs and evicting it back to CPU afterwards. This lowers peak\nGPU memory to roughly the largest single stage rather than the whole\npipeline, letting larger models fit on limited VRAM, at the cost of\nhost<->device copies between stages (mitigated by ``pin_memory``).\n\nTerminology (used consistently across these fields):\n\n- **component**: a named, public sub-model of the pipeline, e.g.\n ``text_encoder``, ``transformer``, ``vae``.\n- **stage**: one step of the offload schedule \u2014 a single component, or a\n group of components that are co-resident on the GPU and run together\n before being evicted. For example, a stage ``[\"text_encoder\", \"transformer\"]``\n keeps both components on GPU until that stage completes. The ordered list\n of stages is set by ``stages``.", "properties": { "enable": { "default": false, "description": "Enable offloading of model components to CPU between pipeline stages.", "status": "prototype", "title": "Enable", "type": "boolean" }, "pin_memory": { "default": true, "description": "Allocate pinned CPU storage for faster host-to-device copies.", "status": "prototype", "title": "Pin Memory", "type": "boolean" }, "stages": { "anyOf": [ { "items": { "anyOf": [ { "type": "string" }, { "items": { "type": "string" }, "type": "array" } ] }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Optional ordered list of stages, named with model-specific public component names. Each entry is a single component, or a list of components that are co-resident on the GPU and run together as one stage. For example, [['text_encoder', 'transformer'], 'vae'] keeps text_encoder and transformer co-resident before moving to vae. Example components: ['text_encoder', 'transformer', 'vae'] for Wan T2V, or ['reasoner', 'generator', 'text_guardrail', 'video_guardrail', 'vae'] for Cosmos3. If omitted, the model chooses default stages.", "status": "prototype", "title": "Stages" } }, "title": "CpuOffloadConfig", "type": "object" }, "CudaGraphConfig": { "additionalProperties": false, "description": "Configuration for CUDA graph capture/replay.\n\nWarmup shapes for CUDA graph pre-capture are configured via\nCompilationConfig (resolutions + num_frames), not here.", "properties": { "enable": { "default": false, "status": "prototype", "title": "Enable", "type": "boolean" } }, "title": "CudaGraphConfig", "type": "object" }, "ParallelConfig": { "additionalProperties": false, "description": "Configuration for distributed parallelism across DiT-shaped models.\n\nThe sequence axis can be sharded via Attention2D (a 2D mesh) or Ring\nAttention (a 1D mesh); these two are mutually exclusive. Either can be\ncombined with Ulysses head-sharding to form an outer \u00d7 inner sequence\nparallel mesh. See ``mapping.py`` for the underlying DeviceMesh layout.", "properties": { "async_ulysses": { "default": false, "description": "Enable the async Ulysses A2A pipeline: overlap per-rank V/Q/K projection compute with cross-rank symm-mem all-to-all on a dedicated side stream. Requires ulysses_size > 1. Defaults to False.", "status": "prototype", "title": "Async Ulysses", "type": "boolean" }, "attn2d_size": { "default": [ 1, 1 ], "description": "Attention2D context parallelism as (row_size, col_size). (1, 1) disables Attention2D; both entries must be >= 1. A 2x2 mesh gathers Q across the row group and K/V across the column group. Mutually exclusive with ring_size > 1.", "maxItems": 2, "minItems": 2, "prefixItems": [ { "minimum": 1, "type": "integer" }, { "minimum": 1, "type": "integer" } ], "status": "prototype", "title": "Attn2D Size", "type": "array" }, "cfg_size": { "default": 1, "description": "CFG (classifier-free guidance) batch parallelism degree. cfg_size=2 splits positive/negative prompts across 2 GPUs.", "maximum": 2, "minimum": 1, "status": "prototype", "title": "Cfg Size", "type": "integer" }, "parallel_vae_size": { "default": 1, "description": "Number of ranks used for VAE parallelism. 1 disables parallel VAE.", "minimum": 1, "status": "prototype", "title": "Parallel Vae Size", "type": "integer" }, "parallel_vae_split_dim": { "default": "width", "enum": [ "width", "height" ], "status": "prototype", "title": "Parallel Vae Split Dim", "type": "string" }, "ring_size": { "default": 1, "description": "Ring Attention sequence-parallel degree. The sequence is sharded across ring_size GPUs and K/V blocks stream around the ring. Mutually exclusive with attn2d_size > (1, 1); combinable with ulysses_size.", "minimum": 1, "status": "prototype", "title": "Ring Size", "type": "integer" }, "tp_layout": { "anyOf": [ { "enum": [ "replicated", "token_sharded" ], "type": "string" }, { "type": "null" } ], "default": null, "description": "Activation layout between TP projections: 'replicated' (all-reduce TP; the default when unset) or 'token_sharded' (each TP rank holds 1/tp_size of the tokens; each all-reduce becomes a reduce-scatter plus an all-gather). 'token_sharded' requires tp_size > 1 and no ulysses/ring/attn2d.", "status": "prototype", "title": "Tp Layout" }, "tp_size": { "default": 1, "description": "Tensor parallel group size. Heads are sharded across tp_size GPUs.", "minimum": 1, "status": "prototype", "title": "Tp Size", "type": "integer" }, "ulysses_size": { "default": 1, "description": "Ulysses head-sharding degree. Heads are sharded across ulysses_size GPUs.", "minimum": 1, "status": "prototype", "title": "Ulysses Size", "type": "integer" } }, "title": "ParallelConfig", "type": "object" }, "QuantAlgo": { "enum": [ "W8A16", "W4A16", "W4A16_AWQ", "W4A8_AWQ", "W8A16_GPTQ", "W4A16_GPTQ", "W8A8_SQ_PER_CHANNEL", "W8A8_SQ_PER_TENSOR_PLUGIN", "W8A8_SQ_PER_CHANNEL_PER_TOKEN_PLUGIN", "W8A8_SQ_PER_CHANNEL_PER_TENSOR_PLUGIN", "W8A8_SQ_PER_TENSOR_PER_TOKEN_PLUGIN", "W4A8_QSERVE_PER_GROUP", "W4A8_QSERVE_PER_CHANNEL", "FP8", "FP8_PER_CHANNEL_PER_TOKEN", "FP8_BLOCK_SCALES", "INT8", "MIXED_PRECISION", "NVFP4", "W4A8_NVFP4_FP8", "W4A8_MXFP4_FP8", "W4A8_MXFP4_MXFP8", "W4A16_MXFP4", "MXFP8", "W4A16_NVFP4", "NVFP4_AWQ", "NVFP4_ARC", "NO_QUANT" ], "title": "QuantAlgo", "type": "string" }, "QuantAttentionConfig": { "additionalProperties": false, "description": "Attention quantization recipe (TRTLLM / CUDNN / FLASHINFER / CUTEDSL backends).\n\nSpecifies Q/K and V quantization formats and their optional block sizes.\n\nBare QuantAttentionConfig() uses the Qk16Pv8 defaults. Recipe validity is\nbackend-specific; FLASHINFER supports block-scaled MXFP8 or NVFP4 Q/K with FP8 V\non SM100/SM103, or NVFP4 Q/K/V on SM120/SM121. Unsupported recipes are rejected\nby AttentionConfig's validator with a ValueError.", "properties": { "k_block_size": { "default": 0, "description": "Number of K tokens per SageAttention quantization block; 0 otherwise.", "minimum": 0, "status": "prototype", "title": "K Block Size", "type": "integer" }, "q_block_size": { "default": 0, "description": "Number of Q tokens per SageAttention quantization block; 0 otherwise.", "minimum": 0, "status": "prototype", "title": "Q Block Size", "type": "integer" }, "qk_dtype": { "default": "bf16", "description": "Q/K quantization format. bf16 leaves Q/K unquantized; int8 and fp8 use 8-bit integer and floating-point element formats; mxfp8 and nvfp4 are block-scaled formats.", "enum": [ "bf16", "int8", "fp8", "mxfp8", "nvfp4" ], "status": "prototype", "title": "Qk Dtype", "type": "string" }, "smooth_k": { "default": false, "description": "Subtract the per-channel K mean before quantizing K, which narrows the range the quantized type has to cover. SageAttention only.", "status": "prototype", "title": "Smooth K", "type": "boolean" }, "v_block_size": { "default": 0, "description": "V quantization block size on the hidden dimension; 0 uses one tensor-wide V scale.", "minimum": 0, "status": "prototype", "title": "V Block Size", "type": "integer" }, "v_dtype": { "default": "fp8", "description": "V quantization format. fp8 is the 8-bit floating-point element format; mxfp8 and nvfp4 are block-scaled formats.", "enum": [ "fp8", "mxfp8", "nvfp4" ], "status": "prototype", "title": "V Dtype", "type": "string" } }, "title": "QuantAttentionConfig", "type": "object" }, "QuantConfig": { "additionalProperties": false, "description": "Serializable quantization configuration class, part of the PretrainedConfig.", "properties": { "clamp_val": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Clamp values used in FP8 rowwise quantization.", "title": "Clamp Val" }, "exclude_modules": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Module name patterns that are skipped in quantization.", "title": "Exclude Modules" }, "group_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "default": 128, "description": "Group size for group-wise quantization.", "title": "Group Size" }, "has_zero_point": { "default": false, "description": "Whether to use zero point for quantization.", "title": "Has Zero Point", "type": "boolean" }, "kv_cache_quant_algo": { "anyOf": [ { "$ref": "#/$defs/QuantAlgo" }, { "type": "null" } ], "default": null, "description": "KV cache quantization algorithm." }, "mamba_ssm_cache_dtype": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "Data type for mamba SSM cache.", "title": "Mamba Ssm Cache Dtype" }, "mamba_ssm_philox_rounds": { "default": 10, "description": "Number of Philox rounds for stochastic rounding PRNG. Higher values give better randomness.", "minimum": 1, "title": "Mamba Ssm Philox Rounds", "type": "integer" }, "mamba_ssm_stochastic_rounding": { "default": false, "description": "Enable stochastic rounding for Mamba SSM state updates. Requires fp16 cache.", "title": "Mamba Ssm Stochastic Rounding", "type": "boolean" }, "pre_quant_scale": { "default": false, "description": "Whether to use pre-quant scale for quantization.", "title": "Pre Quant Scale", "type": "boolean" }, "quant_algo": { "anyOf": [ { "$ref": "#/$defs/QuantAlgo" }, { "type": "null" } ], "default": null, "description": "Quantization algorithm.", "telemetry": true }, "smoothquant_val": { "default": 0.5, "description": "Smoothing parameter alpha used in smooth quant.", "title": "Smoothquant Val", "type": "number" }, "use_meta_recipe": { "default": false, "description": "Whether to use Meta's recipe for FP8 rowwise quantization.", "title": "Use Meta Recipe", "type": "boolean" } }, "title": "QuantConfig", "type": "object" }, "RuntimeLoRAConfig": { "additionalProperties": false, "description": "Startup-preloaded LoRA adapter fused into transformer weights.", "properties": { "fuse_qkv": { "default": true, "description": "Map to_q/to_k/to_v LoRA tensors onto fused qkv_proj modules when present.", "status": "prototype", "title": "Fuse Qkv", "type": "boolean" }, "key_map": { "additionalProperties": { "type": "string" }, "description": "Prefix map from LoRA checkpoint module names to TRTLLM module names.", "status": "prototype", "title": "Key Map", "type": "object" }, "path": { "default": "", "description": "Local safetensors file or directory containing LoRA adapter weights.", "status": "prototype", "title": "Path", "type": "string" }, "scale": { "default": 1.0, "description": "LoRA adapter strength multiplier applied on top of alpha/rank scaling.", "status": "prototype", "title": "Scale", "type": "number" }, "strict": { "default": true, "description": "Raise when adapter tensors do not match any target module or expected shapes.", "status": "prototype", "title": "Strict", "type": "boolean" }, "strip_prefixes": { "description": "Extra LoRA key prefixes to strip before matching transformer module names.", "items": { "type": "string" }, "status": "prototype", "title": "Strip Prefixes", "type": "array" }, "target_components": { "description": "Transformer component names to patch. Empty is allowed only for single-transformer pipelines; multi-transformer pipelines require explicit component names.", "items": { "type": "string" }, "status": "prototype", "title": "Target Components", "type": "array" } }, "title": "RuntimeLoRAConfig", "type": "object" }, "SkipSoftmaxAttentionConfig": { "additionalProperties": false, "description": "SkipSoftmax sparse attention configuration for visual generation.\n\nThis class is only the Python/YAML user surface. Checkpoint calibration\ndata is passed to ``to_sparse_params()`` when SparseParams are created.", "properties": { "algorithm": { "const": "skip_softmax", "default": "skip_softmax", "title": "Algorithm", "type": "string" }, "disabled_until_timestep": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Normalized timestep cutoff below which skip-softmax is enabled.", "title": "Disabled Until Timestep" }, "target_sparsity": { "anyOf": [ { "maximum": 1.0, "minimum": 0.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Semantic target sparsity in [0, 1]; requires a calibration formula.", "title": "Target Sparsity" }, "threshold_scale_factor": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "description": "Raw per-block threshold; takes precedence over target_sparsity.", "title": "Threshold Scale Factor" } }, "title": "SkipSoftmaxAttentionConfig", "type": "object" }, "SolAttentionConfig": { "additionalProperties": false, "description": "SOL sparse attention configuration for visual generation.\n\nSOL (Sol-Attn, arXiv:2607.24027) routes attention blocks dynamically and\ncorrects the approximation of the blocks it skips. Two backends serve it:\n\n- ``TRTLLM`` runs SOL in two stages. A TRT-LLM-owned predictor derives an\n exact block bitmask and K/V proxy summaries from Q/K/V, then the generic\n PrimTS block-sparse FMHA executes that route. Unsupported runtime tensor\n envelopes raise instead of silently falling back to dense attention.\n- ``CUTEDSL`` runs the fused kernel vendored from the reference\n implementation, which folds routing, sparse computation and correction\n into one online-softmax pass on datacenter Blackwell (sm100/sm103) with\n head_dim=128, bf16 and MHA. Inputs the kernel cannot serve fall back to\n dense attention and are counted; GQA/MQA raises at construction.\n\nContext parallelism (cp_size > 1) is rejected for both backends in\nvisual_gen/attention/attention.py.", "properties": { "algorithm": { "const": "sol_attn", "default": "sol_attn", "title": "Algorithm", "type": "string" }, "dense_layers": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Layer indices forced dense regardless of the dense prefix, e.g. [0, 2, 3]. Evaluated per layer at construction time; no pipeline wiring required.", "title": "Dense Layers" }, "disabled_until_timestep": { "anyOf": [ { "exclusiveMinimum": 0.0, "maximum": 1.0, "type": "number" }, { "type": "null" } ], "default": null, "description": "Dense-prefix cutoff on the normalized denoising timestep, with the same sense as skip_softmax's field of the same name: the layer runs dense while timestep >= this value and switches to SOL below it. Larger timesteps are earlier, noisier steps, so this protects the high-noise prefix. Use None (not 0.0) to disable the prefix; 0.0 is rejected because it would run dense on every step and silently turn SOL off entirely. Read from the `timestep` forward kwarg, which must be the normalized scheduler time (larger = noisier); a pipeline that does not pass it runs sparse on every step (fail-open).", "title": "Disabled Until Timestep" }, "tau": { "default": 1.0, "description": "Routing threshold in standard deviations above the mean block score; higher tau routes more blocks sparse.", "title": "Tau", "type": "number" }, "thresh_type": { "default": "diag", "description": "Block routing threshold policy: 'diag' models each key channel independently, 'exact' uses the full key covariance.", "enum": [ "diag", "exact" ], "title": "Thresh Type", "type": "string" } }, "title": "SolAttentionConfig", "type": "object" }, "TeaCacheConfig": { "additionalProperties": false, "description": "TeaCache step-caching acceleration config.", "properties": { "cache_backend": { "const": "teacache", "default": "teacache", "title": "Cache Backend", "type": "string" }, "coefficients": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Polynomial coefficients used by the TeaCache decision function. None (default) uses the pipeline's built-in per-checkpoint table; an explicit list overrides the table entirely.", "status": "prototype", "title": "Coefficients" }, "coefficients_2": { "anyOf": [ { "items": { "type": "number" }, "type": "array" }, { "type": "null" } ], "default": null, "description": "Second polynomial (Wan 2.2 dual-transformer low-noise stage only). Required together with coefficients when enabling TeaCache on Wan 2.2.", "status": "prototype", "title": "Coefficients 2" }, "teacache_thresh": { "default": 0.2, "exclusiveMinimum": 0.0, "status": "prototype", "title": "Teacache Thresh", "type": "number" }, "use_ret_steps": { "default": false, "status": "prototype", "title": "Use Ret Steps", "type": "boolean" } }, "title": "TeaCacheConfig", "type": "object" }, "TorchCompileConfig": { "additionalProperties": false, "description": "Configuration for torch.compile and autotuning.\n\nWarmup shapes for torch.compile specialization are configured via\nCompilationConfig (resolutions + num_frames), not here.", "properties": { "enable": { "default": true, "status": "prototype", "title": "Enable", "type": "boolean" }, "enable_autotune": { "default": true, "status": "prototype", "title": "Enable Autotune", "type": "boolean" }, "enable_fullgraph": { "default": false, "status": "prototype", "title": "Enable Fullgraph", "type": "boolean" } }, "title": "TorchCompileConfig", "type": "object" }, "VAEConfig": { "additionalProperties": false, "description": "Configuration for the variational autoencoder.", "properties": { "quant_conv_config": { "anyOf": [ { "$ref": "#/$defs/QuantConfig" }, { "additionalProperties": true, "type": "object" }, { "type": "null" } ], "default": null, "description": "Quantization config for VAE convolution operators, independent from transformer linear-layer and attention quantization.", "status": "prototype", "title": "Quant Conv Config" } }, "title": "VAEConfig", "type": "object" }, "VideoSparseAttentionConfig": { "additionalProperties": false, "description": "Video Sparse Attention (VSA) sparse-attention recipe.\n\nTwo-stage hybrid attention: a coarse mean-pooled stage over (4,4,4) cubes\nand a block-sparse fine stage over the top-K cubes selected per head.\nvsa_sparsity controls the fraction of cubes dropped on the fine stage.\nThe fine stage may run on either the CuTeDSL backend or the TRTLLM PrimTS\nbackend, while the user-facing sparsity semantics stay the same.", "properties": { "algorithm": { "const": "vsa", "default": "vsa", "description": "Sparse attention algorithm discriminator.", "title": "Algorithm", "type": "string" }, "vsa_sparsity": { "default": 0.9, "description": "Fraction of cubes dropped on the fine stage. 0.0 keeps all cubes (dense fine stage); values closer to 1.0 keep fewer cubes.", "maximum": 1.0, "minimum": 0.0, "title": "Vsa Sparsity", "type": "number" } }, "title": "VideoSparseAttentionConfig", "type": "object" } }, "$id": "https://nvidia.github.io/TensorRT-LLM/1.4.0rc1/_static/schemas/trtllm-serve-visual-gen-config.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, "description": "Static YAML checks only; runtime validation remains authoritative.", "properties": { "attention_config": { "$ref": "#/$defs/AttentionConfig", "status": "prototype" }, "cache_config": { "anyOf": [ { "discriminator": { "mapping": { "cache_dit": "#/$defs/CacheDiTConfig", "teacache": "#/$defs/TeaCacheConfig" }, "propertyName": "cache_backend" }, "oneOf": [ { "$ref": "#/$defs/TeaCacheConfig" }, { "$ref": "#/$defs/CacheDiTConfig" } ] }, { "type": "null" } ], "default": null, "status": "prototype", "title": "Cache Config" }, "compilation_config": { "$ref": "#/$defs/CompilationConfig", "status": "prototype" }, "cpu_offload_config": { "$ref": "#/$defs/CpuOffloadConfig", "status": "prototype" }, "cuda_graph_config": { "$ref": "#/$defs/CudaGraphConfig", "status": "prototype" }, "enable_layerwise_nvtx_marker": { "default": false, "description": "If True, register per-layer NVTX hooks at load time for nsys profile readability.", "status": "prototype", "title": "Enable Layerwise Nvtx Marker", "type": "boolean" }, "model": { "default": "", "description": "Local directory path or HuggingFace Hub model ID (e.g., 'Wan-AI/Wan2.1-T2V-1.3B-Diffusers'). Hub models are downloaded and cached automatically.", "status": "prototype", "title": "Model", "type": "string" }, "parallel_config": { "$ref": "#/$defs/ParallelConfig", "status": "prototype" }, "pipeline_config": { "additionalProperties": true, "description": "Per-architecture pipeline runtime knobs. Strict \u2014 unknown keys raise at load time. See VisualGen.pipeline_config(model) for the legal key set for a given model.", "status": "prototype", "title": "Pipeline Config", "type": "object" }, "quant_config": { "anyOf": [ { "$ref": "#/$defs/QuantConfig" }, { "additionalProperties": true, "type": "object" } ], "description": "Quantization config \u2014 accepts either a QuantConfig instance or a ModelOpt-format dict (e.g. ``{'quant_algo': 'FP8', 'dynamic': True}``). Dict-form parsing happens lazily in DiffusionPipelineConfig.from_pretrained.", "status": "prototype", "title": "Quant Config" }, "revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "description": "HuggingFace Hub revision (branch, tag, or commit SHA) to download.", "status": "prototype", "title": "Revision" }, "runtime_lora_config": { "anyOf": [ { "$ref": "#/$defs/RuntimeLoRAConfig" }, { "type": "null" } ], "default": null, "description": "Startup-preloaded LoRA adapter fused into VisualGen transformer weights.", "status": "prototype" }, "torch_compile_config": { "$ref": "#/$defs/TorchCompileConfig", "status": "prototype" }, "vae_config": { "$ref": "#/$defs/VAEConfig", "description": "Configuration for VAE execution.", "status": "prototype" } }, "title": "TensorRT-LLM 1.4.0rc1 trtllm-serve --visual_gen_args", "type": "object" }