Decoding Strategy#

class DecodingStrategy#

Subclassed by trt_edgellm::rt::DFlashDecoder, trt_edgellm::rt::EagleDecoder, trt_edgellm::rt::Gemma4MTPDecoder, trt_edgellm::rt::MTPDecoder, trt_edgellm::rt::VanillaDecoder

Public Functions

virtual ~DecodingStrategy() noexcept = default#

virtual DecodingStrategyKind kind() const noexcept = 0#

virtual char const *name() const noexcept = 0#

virtual bool isSpeculative() const noexcept = 0#

virtual bool decodeStep(DecodingInferenceContext &context) = 0#

virtual bool captureCudaGraphs(cudaStream_t stream) = 0#

virtual int64_t getRequiredContextMemorySize() const noexcept = 0#

virtual void setContextMemory(Tensor&) = 0#

virtual bool hasSystemPromptKVCache( SystemPromptCacheKey const&, ) const = 0#

virtual void restoreSystemPromptKVCache( SystemPromptCacheKey const&, int32_t, cudaStream_t, ) = 0#

virtual bool runSystemPromptPrefill(DecodingInferenceContext&) = 0#

virtual void saveSystemPromptKVCache( SystemPromptCacheKey const&, std::string const&, std::vector<tokenizer::Rank> const&, int32_t, cudaStream_t, ) = 0#

virtual void resetForNewSequences(Tensor&, cudaStream_t) = 0#

virtual void onBatchEvict( std::vector<int32_t> const&, int32_t, int32_t, Tensor&, cudaStream_t, ) = 0#

struct SamplingBuffers#

Public Members

Tensor &workspace#

Tensor &indices#

Tensor &scores#

Tensor &baseVocabMappingTable#

Tensor &hostPackedTokenIds#

Tensor &hostSelectedTokenIds#

struct LogprobsBuffers#

GPU/CPU buffers for logprobs computation (owned by LLMInferenceRuntime). Referenced here so decoders can call log-softmax + top-K on the output logits.

Public Members

Tensor &deviceLogprobsValues#: GPU [logprobsMaxBatch, kMaxLogprobsK] top-K log-prob values.

Tensor &deviceLogprobsIndices#: GPU [logprobsMaxBatch, kMaxLogprobsK] top-K token indices.

Tensor &hostLogprobsValues#: CPU pinned [logprobsMaxBatch, kMaxLogprobsK].

Tensor &hostLogprobsIndices#: CPU pinned [logprobsMaxBatch, kMaxLogprobsK]

Tensor &gatheredLogits#: GPU [maxBatch * maxAcceptDepth, vocab] accepted verify rows gathered before extraction. Used by the spec-decode verify paths whose accepted rows are non-contiguous in the output logits (EAGLE / MTP / DFlash); Gemma4 MTP’s sequential chain reads logits directly.

struct BaseEngineResources#

Base-engine execution infrastructure: executor, tensor map, KV cache, pipeline I/O, shared resources, and CUDA-graph capture callback.

Public Members