CompletionOutput#

class tensorrt_llm.llmapi.CompletionOutput(
index: int,
text: str = '',
token_ids: ~typing.List[int] | None = <factory>,
cumulative_logprob: float | None = None,
logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
prompt_logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
finish_reason: ~typing.Literal['stop',
'length',
'timeout',
'cancelled'] | None = None,
stop_reason: int | str | None = None,
generation_logits: ~torch.Tensor | None = None,
additional_context_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
additional_generation_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
disaggregated_params: ~tensorrt_llm.disaggregated_params.DisaggregatedParams | None = None,
request_perf_metrics: ~tensorrt_llm.bindings.executor.RequestPerfMetrics | None = None,
_postprocess_result: ~typing.Any = None,
)[source]#

Bases: object

The output data of one completion output of a request.

Parameters:

  • index (int) – The index of the output in the request.

  • text (str) – The generated output text. Defaults to “”.

  • token_ids (List[int], optional) – The token ids of the generated output text. Defaults to [].

  • cumulative_logprob (float, optional) – The cumulative log probability of the generated output text. Defaults to None.

  • logprobs (TokenLogprobs | SimpleTokenLogprobs, optional) – The log probabilities of the top probability words at each position if the logprobs are requested. Defaults to None.

  • prompt_logprobs (TokenLogprobs | SimpleTokenLogprobs, optional) – The log probabilities per prompt token. Defaults to None.

  • finish_reason (Literal['stop', 'length', 'timeout', 'cancelled'], optional) – The reason why the sequence is finished. Defaults to None.

  • stop_reason (int, str, optional) – The stop string or token id that caused the completion to stop, None if the completion finished for some other reason. Defaults to None.

  • generation_logits (torch.Tensor, optional) – The logits on the generated output token ids. Defaults to None.

  • additional_context_outputs (Dict[str, torch.Tensor], optional) – The additional context outputs. Defaults to None.

  • additional_generation_outputs (Dict[str, torch.Tensor], optional) – The additional generation outputs. Defaults to None.

  • disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, optional) – Parameters needed for disaggregated serving. Includes the type of request, the first generated tokens, the context request id and the any additional state needing to be transferred from context and generation instances. Defaults to None.

  • request_perf_metrics (tensorrt_llm.bindings.executor.RequestPerfMetrics, optional) – Performance metrics for the request. Defaults to None.

length#

The number of generated tokens.

Type:

int

token_ids_diff#

Newly generated token ids.

Type:

List[int]

logprobs_diff#

Logprobs of newly generated tokens.

Type:

TokenLogprobs | SimpleTokenLogprobs

text_diff#

Newly generated tokens.

Type:

str

__init__(
index: int,
text: str = '',
token_ids: ~typing.List[int] | None = <factory>,
cumulative_logprob: float | None = None,
logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
prompt_logprobs: list[dict[int,
~tensorrt_llm.executor.result.Logprob]] | list[float] | None = <factory>,
finish_reason: ~typing.Literal['stop',
'length',
'timeout',
'cancelled'] | None = None,
stop_reason: int | str | None = None,
generation_logits: ~torch.Tensor | None = None,
additional_context_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
additional_generation_outputs: ~typing.Dict[str,
~torch.Tensor] | None = None,
disaggregated_params: ~tensorrt_llm.disaggregated_params.DisaggregatedParams | None = None,
request_perf_metrics: ~tensorrt_llm.bindings.executor.RequestPerfMetrics | None = None,
_postprocess_result: ~typing.Any = None,
) → None#
additional_context_outputs: Dict[str, Tensor] | None#
additional_generation_outputs: Dict[str, Tensor] | None#
cumulative_logprob: float | None#
disaggregated_params: DisaggregatedParams | None#
finish_reason: Literal['stop', 'length', 'timeout', 'cancelled'] | None#
generation_logits: Tensor | None#
index: int#
property length: int#
logprobs: list[dict[int, Logprob]] | list[float] | None#
property logprobs_diff: list[dict[int, Logprob]] | list[float]#
prompt_logprobs: list[dict[int, Logprob]] | list[float] | None#
request_perf_metrics: RequestPerfMetrics | None#
stop_reason: int | str | None#
text: str#
property text_diff: str#
token_ids: List[int] | None#
property token_ids_diff: List[int]#