DisaggregatedParams#

class tensorrt_llm.llmapi.DisaggregatedParams(
*,
request_type: str | None = None,
first_gen_tokens: List[int] | None = None,
first_gen_log_probs: List | None = None,
first_gen_logits: List | None = None,
ctx_request_id: int | None = None,
opaque_state: bytes | None = None,
draft_tokens: List[int] | None = None,
disagg_request_id: int | None = None,
ctx_dp_rank: int | None = None,
ctx_info_endpoint: str | None = None,
schedule_style: DisaggScheduleStyle | None = None,
ctx_usage: Dict[str, Any] | None = None,
multimodal_embedding_handles: List[Dict[str, Any]] | None = None,
multimodal_hashes: List[List[int]] | None = None,
mrope_position_ids_handle: Dict[str, Any] | None = None,
mrope_position_deltas_handle: Dict[str, Any] | None = None,
)[source]#

Bases: object

Disaggregated serving parameters.

Parameters:

  • request_type (str) – The type of request (“context_only” | “generation_only” | “context_and_generation”)

  • first_gen_tokens (List[int]) – The first tokens of the generation request

  • ctx_request_id (int) – The context request id

  • opaque_state (bytes) – Any additional state needing to be exchanged between context and gen instances

  • draft_tokens (List[int]) – The draft tokens of the generation request

  • disagg_request_id (int) – The disaggregated request id, if set, both context and generation requests will use it as underlying request id.

  • first_gen_log_probs (List) – The logprobs for first_gen_tokens, produced during prefill. Each entry is a list (one per beam) of either TokenLogprobs (list[dict[int, Logprob]], default format) or SimpleTokenLogprobs (list[float], simple format).

  • first_gen_logits (List) – The generation logits for first_gen_tokens, produced during prefill. Each entry is a torch.Tensor of shape [num_tokens, vocab_size] (one per beam/sequence).

  • ctx_usage (Dict[str, Any]) – The context usage payload to preserve exact usage accounting on the generation server.

  • multimodal_embedding_handles (List[Dict[str, Any]]) – The resulting multimodal embedding handles from ViT.

  • multimodal_hashes (List[List[int]]) – The multimodal hashes of each multimodal item in the request.

__init__(
*,
request_type: str | None = None,
first_gen_tokens: List[int] | None = None,
first_gen_log_probs: List | None = None,
first_gen_logits: List | None = None,
ctx_request_id: int | None = None,
opaque_state: bytes | None = None,
draft_tokens: List[int] | None = None,
disagg_request_id: int | None = None,
ctx_dp_rank: int | None = None,
ctx_info_endpoint: str | None = None,
schedule_style: DisaggScheduleStyle | None = None,
ctx_usage: Dict[str, Any] | None = None,
multimodal_embedding_handles: List[Dict[str, Any]] | None = None,
multimodal_hashes: List[List[int]] | None = None,
mrope_position_ids_handle: Dict[str, Any] | None = None,
mrope_position_deltas_handle: Dict[str, Any] | None = None,
) → None#
get_context_phase_params() → ContextPhaseParams[source]#
get_request_type() → RequestType[source]#
ctx_dp_rank: int | None#
ctx_info_endpoint: str | None#
ctx_request_id: int | None#
ctx_usage: Dict[str, Any] | None#
disagg_request_id: int | None#
draft_tokens: List[int] | None#
first_gen_log_probs: List | None#
first_gen_logits: List | None#
first_gen_tokens: List[int] | None#
mrope_position_deltas_handle: Dict[str, Any] | None#
mrope_position_ids_handle: Dict[str, Any] | None#
multimodal_embedding_handles: List[Dict[str, Any]] | None#
multimodal_hashes: List[List[int]] | None#
opaque_state: bytes | None#
request_type: str | None#
schedule_style: DisaggScheduleStyle | None#