CompletionOutput#

class tensorrt_llm.llmapi.CompletionOutput(
index: int,
text: str = '',
token_ids: ~typing.List[int] | None = <factory>,
cumulative_logprob: float | None = None,
logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
prompt_logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
finish_reason: ~typing.Literal['stop',
'length',
'timeout',
'cancelled'] | None = None,
stop_reason: int | str | None = None,
generation_logits: ~torch.Tensor | None = None,
additional_context_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
additional_generation_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
disaggregated_params: ~tensorrt_llm.disaggregated_params.DisaggregatedParams | None = None,
request_perf_metrics: ~tensorrt_llm.bindings.executor.RequestPerfMetrics | None = None,
_postprocess_result: ~typing.Any = None,
)[source]#

Bases: object

The output data of one completion output of a request.

Parameters:

  • index (int) – The index of the output in the request.

  • text (str) – The generated output text. Defaults to “”.

  • token_ids (List[int], optional) – The token ids of the generated output text. Defaults to [].

  • cumulative_logprob (float, optional) – The cumulative log probability of the generated output text. Defaults to None.

  • logprobs (TokenLogprobs | SimpleTokenLogprobs, optional) – The log probabilities of the top probability words at each position if the logprobs are requested. Defaults to None.

  • prompt_logprobs (TokenLogprobs | SimpleTokenLogprobs, optional) – The log probabilities per prompt token. Defaults to None.

  • finish_reason (Literal['stop', 'length', 'timeout', 'cancelled'], optional) – The reason why the sequence is finished. Defaults to None.

  • stop_reason (int, str, optional) – The stop string or token id that caused the completion to stop, None if the completion finished for some other reason. Defaults to None.

  • generation_logits (torch.Tensor, optional) – The logits on the generated output token ids. Defaults to None.

  • additional_context_outputs (Dict[str, torch.Tensor], optional) – The additional context outputs. Defaults to None.

  • additional_generation_outputs (Dict[str, torch.Tensor], optional) – The additional generation outputs. Defaults to None.

  • disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, optional) – Parameters needed for disaggregated serving. Includes the type of request, the first generated tokens, the context request id and the any additional state needing to be transferred from context and generation instances. Defaults to None.

  • request_perf_metrics (tensorrt_llm.bindings.executor.RequestPerfMetrics, optional) – Performance metrics for the request. Defaults to None.

length#

The number of generated tokens.

Type:

int

token_ids_diff#

Newly generated token ids.

Type:

List[int]

logprobs_diff#

Logprobs of newly generated tokens.

Type:

TokenLogprobs | SimpleTokenLogprobs

text_diff#

Newly generated tokens.

Type:

str

routed_experts#

Per-token pre-EPLB logical top-k MoE expert ids (Router Replay / R3), or None when not requested.

Type:

Optional[torch.Tensor]

__init__(
index: int,
text: str = '',
token_ids: ~typing.List[int] | None = <factory>,
cumulative_logprob: float | None = None,
logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
prompt_logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
finish_reason: ~typing.Literal['stop',
'length',
'timeout',
'cancelled'] | None = None,
stop_reason: int | str | None = None,
generation_logits: ~torch.Tensor | None = None,
additional_context_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
additional_generation_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
disaggregated_params: ~tensorrt_llm.disaggregated_params.DisaggregatedParams | None = None,
request_perf_metrics: ~tensorrt_llm.bindings.executor.RequestPerfMetrics | None = None,
_postprocess_result: ~typing.Any = None,
) → None#
additional_context_outputs: Dict[str, Tensor] | None#
additional_generation_outputs: Dict[str, Tensor] | None#
cumulative_logprob: float | None#
disaggregated_params: DisaggregatedParams | None#
finish_reason: Literal['stop', 'length', 'timeout', 'cancelled'] | None#
generation_logits: Tensor | None#
index: int#
property length: int#
logprobs: list[dict[int, Logprob]] | list[float] | None#
property logprobs_diff: list[dict[int, Logprob]] | list[float]#
prompt_logprobs: list[dict[int, Logprob]] | list[float] | None#
request_perf_metrics: RequestPerfMetrics | None#
property routed_experts: Tensor | None#

Per-token pre-EPLB logical top-k MoE expert ids (Router Replay / R3).

Shape [seq_len - 1, num_moe_layers, top_k] when requested via SamplingParams.return_routed_experts (with the engine-level enable_return_routed_experts); None otherwise. Surfaced from additional_generation_outputs["routed_experts"].

stop_reason: int | str | None#
text: str#
property text_diff: str#
token_ids: List[int] | None#
property token_ids_diff: List[int]#