KvCacheConfig#

class tensorrt_llm.llmapi.KvCacheConfig( *, enable_block_reuse: bool = True, max_tokens: int | None = None, max_attention_window: ~typing.Annotated[~typing.List[int] | None, ~annotated_types.MinLen(min_length=1)] = None, sink_token_length: int | None = None, free_gpu_memory_fraction: ~typing.Annotated[float | None, ~annotated_types.Ge(ge=0), ~annotated_types.Le(le=1)] = 0.9, host_cache_size: int | None = None, disk_cache_size: ~typing.Annotated[int, ~annotated_types.Ge(ge=0)] | None = None, disk_cache_path: str | None = None, cross_kv_cache_fraction: float | None = None, secondary_offload_min_priority: int | None = None, event_buffer_max_size: int = 0, attention_dp_events_gather_period_ms: int = 5, enable_partial_reuse: bool = True, copy_on_partial_reuse: bool = True, use_uvm: bool = False, max_gpu_total_bytes: ~typing.Annotated[int, ~annotated_types.Ge(ge=0)] = 0, iteration_stats_interval: ~typing.Annotated[int, ~annotated_types.Gt(gt=0)] = 1, dtype: str = 'auto', mamba_ssm_cache_dtype: ~typing.Literal['auto', 'float16', 'bfloat16', 'float32'] = 'auto', mamba_ssm_stochastic_rounding: bool = False, mamba_ssm_philox_rounds: ~typing.Annotated[int, ~annotated_types.Ge(ge=1)] = 10, tokens_per_block: int = 32, mamba_state_cache_interval: ~typing.Annotated[int, ~annotated_types.Ge(ge=0)] | None = None, mamba_state_config: ~tensorrt_llm.llmapi.llm_args.MambaStateConfig = <factory>, use_kv_cache_manager_v2: bool | ~typing.Literal['auto'] = 'auto', enable_swa_scratch_reuse: bool = False, kv_cache_event_hash_algo: ~typing.Literal['auto', 'v1_block_key', 'v2_sha256', 'v2_sha256_64'] = 'auto', max_util_for_resume: ~typing.Annotated[float, ~annotated_types.Gt(gt=0), ~annotated_types.Le(le=1)] = 0.95, enable_kv_pool_rebalance: bool = False, disk_prefetch_num_reqs: ~typing.Annotated[int, ~annotated_types.Ge(ge=0)] = 0, fp8_context_mla_kv_len_cap: int | None = None, pool_ratio: ~typing.Annotated[~typing.List[float] | None, ~annotated_types.MinLen(min_length=1)] = None, avg_seq_len: ~typing.Annotated[int, ~annotated_types.Gt(gt=0)] | None = None, block_reuse_config: ~tensorrt_llm.llmapi.llm_args.BlockReuseConfig = <factory>, )[source]#

Bases: StrictBaseModel, PybindMirror

Configuration for the KV cache.

field attention_dp_events_gather_period_ms: int = 5#: The period in milliseconds to gather attention DP events across ranks.

field avg_seq_len: Annotated[int, Gt(gt=0)] | None = None#: Average total sequence length of the serving workload, used to build the KV cache manager v2 typical step for hybrid Mamba models and DeepSeek-V4. Hybrid Mamba models warn and fall back to half of max_seq_len when this is unset. This does not take effect when pool_ratio is set.

field block_reuse_config: BlockReuseConfig [Optional]#: KV cache manager v2 configuration for block reuse policies.

field copy_on_partial_reuse: bool = True#: Whether partially matched blocks that are in use can be reused after copying them.

field cross_kv_cache_fraction: float | None = None#: The fraction of the KV Cache memory should be reserved for cross attention. If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Defaults to None (unset); must be set when using an encoder-decoder model and must not be set otherwise.

field disk_cache_path: str | None = None#: Directory used for disk KV cache files. Must be set when disk_cache_size is positive.

field disk_cache_size: Annotated[int, Ge(ge=0)] | None = None#: Size of the disk cache in bytes. Only used by KV cache manager v2 in the PyTorch backend.

field disk_prefetch_num_reqs: int = 0#

Number of queued context requests to prefetch disk-tier KV cache blocks to host for. Set to 0 to disable prefetch. Only effective with KV cache manager v2 and block reuse enabled.

Constraints:

ge = 0

field dtype: str = 'auto'#: The data type for the KV cache. ‘auto’ (default) leaves the checkpoint’s own KV-cache quantization metadata untouched (quant_config.kv_cache_quant_algo is inherited as-is); ‘fp8’, ‘fp8_ds_mla’, or ‘nvfp4’ override it explicitly. ‘fp8_ds_mla’ selects the packed FP8 cache used by sparse MLA on SM90/SM120/SM121. Resolved at LLM-construction time, including when set via trtllm-serve –extra_llm_api_options.

field enable_block_reuse: bool = True#: Controls if KV cache blocks can be reused for different requests.

field enable_kv_pool_rebalance: bool = False#: Opt in to the KVCacheManagerV2 auto-tuner (adjust()) for rebalancing pool-group ratios between iterations. When True the PyExecutor calls adjust() opportunistically; the auto-tuner itself remains gated by V2’s internal 2000-sample / 120s cooldown. When False (default) the rebalance hook is skipped entirely and pool ratios remain at their warmup-derived values. Beta: enable at your own risk. Only used when using KV cache manager v2 (experimental). This option is incompatible with dtype=’fp8_ds_mla’.

field enable_partial_reuse: bool = True#: Whether blocks that are only partially matched can be reused.

field enable_swa_scratch_reuse: bool = False#: Whether KV cache manager v2 uses SWA scratch reuse during prefill.

field event_buffer_max_size: int = 0#: Maximum size of the event buffer. If set to 0, the event buffer will not be used.

field fp8_context_mla_kv_len_cap: int | None = None#: Override, in tokens, for the max summed attended-KV length (total_kv_len) per forward step that the fp8 context-MLA attention workspace is reserved and scheduled for. Only affects fp8 context-MLA models (e.g. DeepSeek / Kimi with an fp8 KV cache). None (default) reserves for the never-stall worst case min(max_batch_size, max_num_tokens) * max_seq_len. A smaller value reserves less workspace (freeing KV cache) and defers context requests whose summed attended KV would exceed it; it is floored at max_seq_len and capped at the worst case. Safe at any value – the scheduler enforces it – trading prefill batching under heavy reuse for KV cache capacity.

field free_gpu_memory_fraction: float | None = 0.9#

The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both max_tokens and free_gpu_memory_fraction are specified, memory corresponding to the minimum will be used.

Constraints:

ge = 0
le = 1

field host_cache_size: int | None = None#: Size of the host cache in bytes. If both max_tokens and host_cache_size are specified, memory corresponding to the minimum will be used.

field iteration_stats_interval: Annotated[int, Gt(gt=0)] = 1#

How often (in iterations) to collect per-iteration KV cache statistics. A value of 1 means every iteration; a value of N means every Nth iteration. Between collections, the C++ deltas accumulate, so the reported deltas cover N iterations.

Constraints:

gt = 0

field kv_cache_event_hash_algo: Literal['auto', 'v1_block_key', 'v2_sha256', 'v2_sha256_64'] = 'auto'#: The block hash algorithm used by KV cache manager events. ‘auto’ uses the native hash for each KV cache manager. Explicit V2 hash choices are ignored with a warning by the V1 KV cache manager.

field mamba_ssm_cache_dtype: Literal['auto', 'float16', 'bfloat16', 'float32'] = 'auto'#: The data type to use for the Mamba SSM cache. If set to ‘auto’, the data type will be inferred from the model config.

field mamba_ssm_philox_rounds: int = 10#

Number of Philox rounds for stochastic rounding PRNG. Higher values give better randomness but increase compute cost. Only used when mamba_ssm_stochastic_rounding is enabled.

Constraints:

ge = 1

field mamba_ssm_stochastic_rounding: bool = False#: Enable stochastic rounding for Mamba SSM state updates. Only applicable with float16 cache dtype.

field mamba_state_cache_interval: Annotated[int, Ge(ge=0)] | None = None#: Deprecated alias for mamba_state_config.periodic_snapshot_interval.

field mamba_state_config: MambaStateConfig [Optional]#: Configuration for reusable Mamba state snapshots.

field max_attention_window: List[int] | None = None#

Size of the attention window for each sequence. Only the last tokens will be stored in the KV cache. If the number of elements in max_attention_window is less than the number of layers, max_attention_window will be repeated multiple times to the number of layers.

Constraints:

min_length = 1

field max_gpu_total_bytes: Annotated[int, Ge(ge=0)] = 0#

The maximum size in bytes of GPU memory that can be allocated for the KV cache. If both max_gpu_total_bytes and free_gpu_memory_fraction are specified, memory corresponding to the minimum will be allocated.

Constraints:

ge = 0

field max_tokens: int | None = None#: The maximum number of tokens that should be stored in the KV cache. If both max_tokens and free_gpu_memory_fraction are specified, memory corresponding to the minimum will be used.

field max_util_for_resume: float = 0.95#

The maximum utilization of the KV cache for resume. Default is 95%. Only used when using KV cache manager v2 (experimental).

Constraints:

gt = 0
le = 1

field pool_ratio: List[float] | None = None#

Initial hot-tier byte ratios by layer group for KV cache manager v2. Values map to KVCacheManagerV2 layer-group ID order and must sum to 1.0. Cold tiers preserve the implied slot-count ratios. Hybrid Mamba models and DeepSeek-V4 use this directly, so avg_seq_len does not take effect when this is set.

Constraints:

min_length = 1

field secondary_offload_min_priority: int | None = None#: Only blocks with priority > secondary_offload_min_priority can be offloaded to secondary memory.

field tokens_per_block: int = 32#: The number of tokens per block.

field use_kv_cache_manager_v2: bool | Literal['auto'] = 'auto'#: Whether to use the KV cache manager v2 (experimental). ‘auto’ uses the model-specific preference and falls back to False when the model does not declare one.

field use_uvm: bool = False#: Whether to use UVM for the KV cache.

class Config#

Bases: object

extra = 'forbid'#

__init__(**data: Any) → None#

Create a new model by parsing and validating input data from keyword arguments.

Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be validated to form a valid model.

self is explicitly positional-only to allow self as a field name.

classmethod construct(

_fields_set: set[str] | None = None,

**values: Any,

) → Self#

Returns a copy of the model.

!!! warning “Deprecated”: This method is now deprecated; use model_copy instead.

If you need include or exclude, use:

`python {test="skip" lint="skip"} data = self.model_dump(include=include, exclude=exclude, round_trip=True) data = {**data, **(update or {})} copied = self.model_validate(data) `

Parameters:

include – Optional set or mapping specifying which fields to include in the copied model.
exclude – Optional set or mapping specifying which fields to exclude in the copied model.
update – Optional dictionary of field-value pairs to override field values in the copied model.
deep – If True, the values of fields that are Pydantic models will be deep-copied.

Returns:

A copy of the model with included, excluded and updated fields as specified.

dict( *, include: set[int] | set[str] | Mapping[int, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | Mapping[str, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | None = None, exclude: set[int] | set[str] | Mapping[int, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | Mapping[str, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | None = None, by_alias: bool = False, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, ) → Dict[str, Any]#

validator disable_periodic_mamba_snapshots_for_conversations » all fields[source]#: Use only explicit stable boundaries for conversation reuse.

classmethod from_orm(obj: Any) → Self#

classmethod from_pybind( pybind_instance: PybindMirror, ) → T#

Construct an instance of the given class from the fields in the given pybind class instance.

Parameters:

cls – Type of the class to construct, must be a subclass of pydantic BaseModel
pybind_instance – Instance of the pybind class to construct from its fields

Notes

When a field value is None in the pybind class, but it’s not optional and has a default value in the BaseModel class, it would get the default value defined in the BaseModel class.

Returns:: Instance of the given class, populated with the fields of the given pybind instance

static get_pybind_enum_fields(pybind_class)#: Get all the enum fields from the pybind class.

static get_pybind_variable_fields(config_cls)#: Get all the variable fields from the pybind class.

json(

*,

include: set[int] | set[str] | Mapping[int, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | Mapping[str, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | None = None,

exclude: set[int] | set[str] | Mapping[int, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | Mapping[str, set[int] | set[str] | Mapping[int, IncEx | bool] | Mapping[str, IncEx | bool] | bool] | None = None,

by_alias: bool = False,

exclude_unset: bool = False,

exclude_defaults: bool = False,

exclude_none: bool = False,

encoder: Callable[[Any], Any] | None = PydanticUndefined,

models_as_dict: bool = PydanticUndefined,

**dumps_kwargs: Any,

) → str#

static maybe_to_pybind(ins)#

validator migrate_legacy_mamba_interval » all fields[source]#: Copy the deprecated Mamba interval into its nested replacement.

static mirror_pybind_enum(pybind_class)#: Mirror the enum fields from the pybind class to the Python class.

static mirror_pybind_fields(pybind_class)#

Class decorator that ensures Python class fields mirror those of a C++ class.

Parameters:: pybind_class – The C++ class whose fields should be mirrored
Returns:: A decorator function that validates field mirroring

classmethod model_construct(

_fields_set: set[str] | None = None,

**values: Any,

) → Self#

Creates a new instance of the Model class with validated data.

Creates a new model setting __dict__ and __pydantic_fields_set__ from trusted or pre-validated data. Default values are respected, but no other validation is performed.

!!! note: model_construct() generally respects the model_config.extra setting on the provided model. That is, if model_config.extra == ‘allow’, then all extra passed values are added to the model instance’s __dict__ and __pydantic_extra__ fields. If model_config.extra == ‘ignore’ (the default), then all extra passed values are ignored. Because no validation is performed with a call to model_construct(), having model_config.extra == ‘forbid’ does not result in an error if extra values are passed, but they will be ignored.

Parameters:

_fields_set – A set of field names that were originally explicitly set during instantiation. If provided, this is directly used for the [model_fields_set][pydantic.BaseModel.model_fields_set] attribute. Otherwise, the field names from the values argument will be used.
values – Trusted or pre-validated data dictionary.

Returns:

A new instance of the Model class with validated data.

model_copy( *, update: Mapping[str, Any] | None = None, deep: bool = False, ) → Self#

!!! abstract “Usage Documentation”: [model_copy](../concepts/models.md#model-copy)

Returns a copy of the model.

!!! note: The underlying instance’s [__dict__][object.__dict__] attribute is copied. This might have unexpected side effects if you store anything in it, on top of the model fields (e.g. the value of [cached properties][functools.cached_property]).

Parameters:

update – Values to change/add in the new model. Note: the data is not validated before creating the new model. You should trust this data.
deep – Set to True to make a deep copy of the model.

Returns:

New model instance.

!!! abstract “Usage Documentation”: [model_dump](../concepts/serialization.md#python-mode)

Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

Parameters:

mode – The mode in which to_python should run. If mode is ‘json’, the output will only contain JSON serializable types. If mode is ‘python’, the output may contain non-JSON-serializable Python objects.
include – A set of fields to include in the output.
exclude – A set of fields to exclude from the output.
context – Additional context to pass to the serializer.
by_alias – Whether to use the field’s alias in the dictionary key if defined.
exclude_unset – Whether to exclude fields that have not been explicitly set.
exclude_defaults – Whether to exclude fields that are set to their default value.
exclude_none – Whether to exclude fields that have a value of None.
exclude_computed_fields – Whether to exclude computed fields. While this can be useful for round-tripping, it is usually recommended to use the dedicated round_trip parameter instead.
round_trip – If True, dumped values should be valid as input for non-idempotent types such as Json[T].
warnings – How to handle serialization errors. False/”none” ignores them, True/”warn” logs errors, “error” raises a [PydanticSerializationError][pydantic_core.PydanticSerializationError].
fallback – A function to call when an unknown value is encountered. If not provided, a [PydanticSerializationError][pydantic_core.PydanticSerializationError] error is raised.
serialize_as_any – Whether to serialize fields with duck-typing serialization behavior.
polymorphic_serialization – Whether to use model and dataclass polymorphic serialization for this call.

Returns:

A dictionary representation of the model.

!!! abstract “Usage Documentation”: [model_dump_json](../concepts/serialization.md#json-mode)

Generates a JSON representation of the model using Pydantic’s to_json method.

Parameters:

indent – Indentation to use in the JSON output. If None is passed, the output will be compact.
ensure_ascii – If True, the output is guaranteed to have all incoming non-ASCII characters escaped. If False (the default), these characters will be output as-is.
include – Field(s) to include in the JSON output.
exclude – Field(s) to exclude from the JSON output.
context – Additional context to pass to the serializer.
by_alias – Whether to serialize using field aliases.
exclude_unset – Whether to exclude fields that have not been explicitly set.
exclude_defaults – Whether to exclude fields that are set to their default value.
exclude_none – Whether to exclude fields that have a value of None.
exclude_computed_fields – Whether to exclude computed fields. While this can be useful for round-tripping, it is usually recommended to use the dedicated round_trip parameter instead.
round_trip – If True, dumped values should be valid as input for non-idempotent types such as Json[T].
warnings – How to handle serialization errors. False/”none” ignores them, True/”warn” logs errors, “error” raises a [PydanticSerializationError][pydantic_core.PydanticSerializationError].
fallback – A function to call when an unknown value is encountered. If not provided, a [PydanticSerializationError][pydantic_core.PydanticSerializationError] error is raised.
serialize_as_any – Whether to serialize fields with duck-typing serialization behavior.
polymorphic_serialization – Whether to use model and dataclass polymorphic serialization for this call.

Returns:

A JSON string representation of the model.

classmethod model_json_schema( by_alias: bool = True, ref_template: str = '#/$defs/{model}', schema_generator: type[~pydantic.json_schema.GenerateJsonSchema] = <class 'pydantic.json_schema.GenerateJsonSchema'>, mode: ~typing.Literal['validation', 'serialization'] = 'validation', *, union_format: ~typing.Literal['any_of', 'primitive_type_array'] = 'any_of', ) → dict[str, Any]#

Generates a JSON schema for a model class.

Parameters:

by_alias – Whether to use attribute aliases or not.
ref_template – The reference template.
union_format –
The format to use when combining schemas from unions together. Can be one of:
- ’any_of’: Use the [anyOf](https://json-schema.org/understanding-json-schema/reference/combining#anyOf)
keyword to combine schemas (the default). - ‘primitive_type_array’: Use the [type](https://json-schema.org/understanding-json-schema/reference/type) keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive type (string, boolean, null, integer or number) or contains constraints/metadata, falls back to any_of.
schema_generator – To override the logic used to generate the JSON schema, as a subclass of GenerateJsonSchema with your desired modifications
mode – The mode in which to generate the schema.

Returns:

The JSON schema for the given model class.

classmethod model_parametrized_name( params: tuple[type[Any], ...], ) → str#

Compute the class name for parametrizations of generic classes.

This method can be overridden to achieve a custom naming scheme for generic BaseModels.

Parameters:: params – Tuple of types of the class. Given a generic class Model with 2 type variables and a concrete model Model[str, int], the value (str, int) would be passed to params.
Returns:: String representing the new class where params are passed to cls as type variables.
Raises:: TypeError – Raised when trying to generate concrete names for non-generic models.

model_post_init(context: Any, /) → None#: Override this method to perform additional initialization after __init__ and model_construct. This is useful if you want to do some validation that requires the entire model to be initialized.

classmethod model_rebuild( *, force: bool = False, raise_errors: bool = True, _parent_namespace_depth: int = 2, _types_namespace: MappingNamespace | None = None, ) → bool | None#

Try to rebuild the pydantic-core schema for the model.

This may be necessary when one of the annotations is a ForwardRef which could not be resolved during the initial attempt to build the schema, and automatic rebuilding fails.

Parameters:

force – Whether to force the rebuilding of the model schema, defaults to False.
raise_errors – Whether to raise errors, defaults to True.
_parent_namespace_depth – The depth level of the parent namespace, defaults to 2.
_types_namespace – The types namespace, defaults to None.

Returns:

Returns None if the schema is already “complete” and rebuilding was not required. If rebuilding _was_ required, returns True if rebuilding was successful, otherwise False.

Validate a pydantic model instance.

Parameters:

obj – The object to validate.
strict – Whether to enforce types strictly.
extra – Whether to ignore, allow, or forbid extra data during model validation. See the [extra configuration value][pydantic.ConfigDict.extra] for details.
from_attributes – Whether to extract data from object attributes.
context – Additional context to pass to the validator.
by_alias – Whether to use the field’s alias when validating against the provided input data.
by_name – Whether to use the field’s name when validating against the provided input data.

Raises:

ValidationError – If the object could not be validated.

Returns:

The validated model instance.

!!! abstract “Usage Documentation”: [JSON Parsing](../concepts/json.md#json-parsing)

Validate the given JSON data against the Pydantic model.

Parameters:

json_data – The JSON data to validate.
strict – Whether to enforce types strictly.
extra – Whether to ignore, allow, or forbid extra data during model validation. See the [extra configuration value][pydantic.ConfigDict.extra] for details.
context – Extra variables to pass to the validator.
by_alias – Whether to use the field’s alias when validating against the provided input data.
by_name – Whether to use the field’s name when validating against the provided input data.

Returns:

The validated Pydantic model.

Raises:

ValidationError – If json_data is not a JSON string or the object could not be validated.

Validate the given object with string data against the Pydantic model.

Parameters:

obj – The object containing string data to validate.
strict – Whether to enforce types strictly.
extra – Whether to ignore, allow, or forbid extra data during model validation. See the [extra configuration value][pydantic.ConfigDict.extra] for details.
context – Extra variables to pass to the validator.
by_alias – Whether to use the field’s alias when validating against the provided input data.
by_name – Whether to use the field’s name when validating against the provided input data.

Returns:

The validated Pydantic model.

classmethod parse_file( path: str | Path, *, content_type: str | None = None, encoding: str = 'utf8', proto: DeprecatedParseProtocol | None = None, allow_pickle: bool = False, ) → Self#

classmethod parse_obj(obj: Any) → Self#

classmethod parse_raw( b: str | bytes, *, content_type: str | None = None, encoding: str = 'utf8', proto: DeprecatedParseProtocol | None = None, allow_pickle: bool = False, ) → Self#

static pybind_equals(obj0, obj1)#: Check if two pybind objects are equal.

validator reject_fp8_ds_mla_pool_rebalance » all fields[source]#: Reject resizing while packed sparse-MLA pool views are fixed.

classmethod schema( by_alias: bool = True, ref_template: str = '#/$defs/{model}', ) → Dict[str, Any]#

classmethod schema_json(

*,

by_alias: bool = True,

ref_template: str = '#/$defs/{model}',

**dumps_kwargs: Any,

) → str#

classmethod update_forward_refs(**localns: Any) → None#

classmethod validate(value: Any) → Self#

validator validate_cross_kv_cache_fraction » cross_kv_cache_fraction[source]#

validator validate_disk_cache_config » all fields[source]#

validator validate_dtype » dtype[source]#

validator validate_free_gpu_memory_fraction » free_gpu_memory_fraction[source]#: Validates that the fraction is between 0.0 and 1.0.

validator validate_mamba_snapshot_offsets » all fields[source]#

validator validate_max_attention_window » max_attention_window[source]#

validator validate_max_gpu_total_bytes » max_gpu_total_bytes[source]#

validator validate_pool_ratio » pool_ratio[source]#

model_computed_fields = {}#

model_config: ClassVar[ConfigDict] = {'extra': 'forbid'}#: Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

property model_extra: dict[str, Any] | None#

Get extra fields set during validation.

Returns:: A dictionary of extra fields, or None if config.extra is not set to “allow”.

model_fields = {'attention_dp_events_gather_period_ms': FieldInfo(annotation=int, required=False, default=5, description='The period in milliseconds to gather attention DP events across ranks.'), 'avg_seq_len': FieldInfo(annotation=Union[Annotated[int, Gt], NoneType], required=False, default=None, description='Average total sequence length of the serving workload, used to build the KV cache manager v2 typical step for hybrid Mamba models and DeepSeek-V4. Hybrid Mamba models warn and fall back to half of max_seq_len when this is unset. This does not take effect when pool_ratio is set.', json_schema_extra={'status': 'prototype'}), 'block_reuse_config': FieldInfo(annotation=BlockReuseConfig, required=False, default_factory=BlockReuseConfig, description='KV cache manager v2 configuration for block reuse policies.', json_schema_extra={'status': 'prototype'}), 'copy_on_partial_reuse': FieldInfo(annotation=bool, required=False, default=True, description='Whether partially matched blocks that are in use can be reused after copying them.'), 'cross_kv_cache_fraction': FieldInfo(annotation=Union[float, NoneType], required=False, default=None, description='The fraction of the KV Cache memory should be reserved for cross attention. If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Defaults to None (unset); must be set when using an encoder-decoder model and must not be set otherwise.'), 'disk_cache_path': FieldInfo(annotation=Union[str, NoneType], required=False, default=None, description='Directory used for disk KV cache files. Must be set when `disk_cache_size` is positive.'), 'disk_cache_size': FieldInfo(annotation=Union[Annotated[int, Ge], NoneType], required=False, default=None, description='Size of the disk cache in bytes. Only used by KV cache manager v2 in the PyTorch backend.'), 'disk_prefetch_num_reqs': FieldInfo(annotation=int, required=False, default=0, description='Number of queued context requests to prefetch disk-tier KV cache blocks to host for. Set to 0 to disable prefetch. Only effective with KV cache manager v2 and block reuse enabled.', metadata=[Ge(ge=0)]), 'dtype': FieldInfo(annotation=str, required=False, default='auto', description="The data type for the KV cache. 'auto' (default) leaves the checkpoint's own KV-cache quantization metadata untouched (quant_config.kv_cache_quant_algo is inherited as-is); 'fp8', 'fp8_ds_mla', or 'nvfp4' override it explicitly. 'fp8_ds_mla' selects the packed FP8 cache used by sparse MLA on SM90/SM120/SM121. Resolved at LLM-construction time, including when set via trtllm-serve --extra_llm_api_options.", json_schema_extra={'telemetry': {'kind': 'categorical', 'converter': 'allowlist', 'allowed_values': ['auto', 'float16', 'bfloat16', 'float32', 'fp8', 'fp8_ds_mla', 'nvfp4']}}), 'enable_block_reuse': FieldInfo(annotation=bool, required=False, default=True, description='Controls if KV cache blocks can be reused for different requests.'), 'enable_kv_pool_rebalance': FieldInfo(annotation=bool, required=False, default=False, description="Opt in to the KVCacheManagerV2 auto-tuner (``adjust()``) for rebalancing pool-group ratios between iterations. When True the PyExecutor calls ``adjust()`` opportunistically; the auto-tuner itself remains gated by V2's internal 2000-sample / 120s cooldown. When False (default) the rebalance hook is skipped entirely and pool ratios remain at their warmup-derived values. Beta: enable at your own risk. Only used when using KV cache manager v2 (experimental). This option is incompatible with dtype='fp8_ds_mla'.", json_schema_extra={'status': 'prototype'}), 'enable_partial_reuse': FieldInfo(annotation=bool, required=False, default=True, description='Whether blocks that are only partially matched can be reused.'), 'enable_swa_scratch_reuse': FieldInfo(annotation=bool, required=False, default=False, description='Whether KV cache manager v2 uses SWA scratch reuse during prefill.', json_schema_extra={'status': 'prototype'}), 'event_buffer_max_size': FieldInfo(annotation=int, required=False, default=0, description='Maximum size of the event buffer. If set to 0, the event buffer will not be used.'), 'fp8_context_mla_kv_len_cap': FieldInfo(annotation=Union[int, NoneType], required=False, default=None, description='Override, in tokens, for the max summed attended-KV length (total_kv_len) per forward step that the fp8 context-MLA attention workspace is reserved and scheduled for. Only affects fp8 context-MLA models (e.g. DeepSeek / Kimi with an fp8 KV cache). None (default) reserves for the never-stall worst case min(max_batch_size, max_num_tokens) * max_seq_len. A smaller value reserves less workspace (freeing KV cache) and defers context requests whose summed attended KV would exceed it; it is floored at max_seq_len and capped at the worst case. Safe at any value -- the scheduler enforces it -- trading prefill batching under heavy reuse for KV cache capacity.', json_schema_extra={'status': 'prototype'}), 'free_gpu_memory_fraction': FieldInfo(annotation=Union[float, NoneType], required=False, default=0.9, description='The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.', metadata=[Ge(ge=0), Le(le=1)]), 'host_cache_size': FieldInfo(annotation=Union[int, NoneType], required=False, default=None, description='Size of the host cache in bytes. If both `max_tokens` and `host_cache_size` are specified, memory corresponding to the minimum will be used.'), 'iteration_stats_interval': FieldInfo(annotation=int, required=False, default=1, description='How often (in iterations) to collect per-iteration KV cache statistics. A value of 1 means every iteration; a value of N means every Nth iteration. Between collections, the C++ deltas accumulate, so the reported deltas cover N iterations.', metadata=[Gt(gt=0)]), 'kv_cache_event_hash_algo': FieldInfo(annotation=Literal['auto', 'v1_block_key', 'v2_sha256', 'v2_sha256_64'], required=False, default='auto', description="The block hash algorithm used by KV cache manager events. 'auto' uses the native hash for each KV cache manager. Explicit V2 hash choices are ignored with a warning by the V1 KV cache manager.", json_schema_extra={'status': 'prototype'}), 'mamba_ssm_cache_dtype': FieldInfo(annotation=Literal['auto', 'float16', 'bfloat16', 'float32'], required=False, default='auto', description="The data type to use for the Mamba SSM cache. If set to 'auto', the data type will be inferred from the model config."), 'mamba_ssm_philox_rounds': FieldInfo(annotation=int, required=False, default=10, description='Number of Philox rounds for stochastic rounding PRNG. Higher values give better randomness but increase compute cost. Only used when mamba_ssm_stochastic_rounding is enabled.', metadata=[Ge(ge=1)]), 'mamba_ssm_stochastic_rounding': FieldInfo(annotation=bool, required=False, default=False, description='Enable stochastic rounding for Mamba SSM state updates. Only applicable with float16 cache dtype.'), 'mamba_state_cache_interval': FieldInfo(annotation=Union[Annotated[int, Ge], NoneType], required=False, default=None, description='Deprecated alias for mamba_state_config.periodic_snapshot_interval.', exclude=True, json_schema_extra={'status': 'deprecated', 'telemetry': {'exclude': True}}), 'mamba_state_config': FieldInfo(annotation=MambaStateConfig, required=False, default_factory=MambaStateConfig, description='Configuration for reusable Mamba state snapshots.'), 'max_attention_window': FieldInfo(annotation=Union[List[int], NoneType], required=False, default=None, description='Size of the attention window for each sequence. Only the last tokens will be stored in the KV cache. If the number of elements in `max_attention_window` is less than the number of layers, `max_attention_window` will be repeated multiple times to the number of layers.', metadata=[MinLen(min_length=1)]), 'max_gpu_total_bytes': FieldInfo(annotation=int, required=False, default=0, description='The maximum size in bytes of GPU memory that can be allocated for the KV cache. If both `max_gpu_total_bytes` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be allocated.', metadata=[Ge(ge=0)]), 'max_tokens': FieldInfo(annotation=Union[int, NoneType], required=False, default=None, description='The maximum number of tokens that should be stored in the KV cache. If both `max_tokens` and `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used.'), 'max_util_for_resume': FieldInfo(annotation=float, required=False, default=0.95, description='The maximum utilization of the KV cache for resume. Default is 95%. Only used when using KV cache manager v2 (experimental).', json_schema_extra={'status': 'prototype'}, metadata=[Gt(gt=0), Le(le=1)]), 'pool_ratio': FieldInfo(annotation=Union[List[float], NoneType], required=False, default=None, description='Initial hot-tier byte ratios by layer group for KV cache manager v2. Values map to KVCacheManagerV2 layer-group ID order and must sum to 1.0. Cold tiers preserve the implied slot-count ratios. Hybrid Mamba models and DeepSeek-V4 use this directly, so avg_seq_len does not take effect when this is set.', json_schema_extra={'status': 'prototype'}, metadata=[MinLen(min_length=1)]), 'secondary_offload_min_priority': FieldInfo(annotation=Union[int, NoneType], required=False, default=None, description='Only blocks with priority > secondary_offload_min_priority can be offloaded to secondary memory.'), 'sink_token_length': FieldInfo(annotation=Union[int, NoneType], required=False, default=None, description='Deprecated and ignored on the PyTorch backend. StreamingLLM is not supported by the PyTorch attention kernels — any non-None value has no effect and will be silently dropped before reaching the executor.', deprecated=True), 'tokens_per_block': FieldInfo(annotation=int, required=False, default=32, description='The number of tokens per block.'), 'use_kv_cache_manager_v2': FieldInfo(annotation=Union[bool, Literal['auto']], required=False, default='auto', description="Whether to use the KV cache manager v2 (experimental). 'auto' uses the model-specific preference and falls back to False when the model does not declare one.", json_schema_extra={'status': 'prototype'}), 'use_uvm': FieldInfo(annotation=bool, required=False, default=False, description='Whether to use UVM for the KV cache.')}#

property model_fields_set: set[str]#

Returns the set of fields that have been explicitly set on this model instance.

Returns:

A set of strings representing the fields that have been set,: i.e. that were not filled from defaults.

sink_token_length: int | None#

Read-only data descriptor used to emit a runtime deprecation warning before accessing a deprecated field.

msg#: The deprecation message to be emitted.

wrapped_property#: The property instance if the deprecated field is a computed field, or None.

field_name#: The name of the field being deprecated.