Decoding Strategy#
-
class DecodingStrategy#
Subclassed by trt_edgellm::rt::BlockDiffusionDecoder, trt_edgellm::rt::DFlashDecoder, trt_edgellm::rt::DSparkDecoder, trt_edgellm::rt::EagleDecoder, trt_edgellm::rt::Gemma4MTPDecoder, trt_edgellm::rt::MTPDecoder, trt_edgellm::rt::VanillaDecoder
Public Functions
-
virtual ~DecodingStrategy() noexcept = default#
-
virtual DecodingStrategyKind kind() const noexcept = 0#
-
virtual char const *name() const noexcept = 0#
-
virtual bool isSpeculative() const noexcept = 0#
- inline virtual DecodingStrategyCapabilities capabilities(
-
inline virtual DecodingKvHeadroomPolicy kvHeadroom() const#
KV headroom of the strategy’s decode rounds; the default is the vanilla one-token step.
- inline virtual DecodingTokenStateContract tokenStateContract(
-
virtual bool decodeStep(DecodingInferenceContext &context) = 0#
-
virtual bool captureCudaGraphs(cudaStream_t stream) = 0#
- inline virtual bool initializeForGeneration( )#
Initialize decoder-private generation state after base prefill. Non-speculative strategies are no-ops.
-
inline virtual void beginBatch() noexcept#
A blocking request’s batch is about to be prefilled: forget the draft progress an earlier request left behind. Unlike resetForNewSequences, KV lengths are kept: a context cache may just have restored them.
-
inline virtual void appendRow(int32_t) noexcept#
A sequence joins a running batch at
executionRow(its last row). Size the per-row state now, so an eviction before seatRow() (a cancelled or failed prefill) still compacts every row.
-
inline virtual bool seatRow(DecodingInferenceContext&, int32_t)#
Seat a sequence admitted into a running batch: bring its per-row state to a resident’s between two decode steps, without touching other rows.
Called inside the batch-one projection of the sequence’s prefill, after the base model ran:
contextand every cache manager the strategy drives present the sequence as row zero, with the first generated token appended and the base outputs still in the pipeline buffers.executionRowis the sequence’s row in the full batch, where its per-row state belongs. A speculative strategy runs the draft prefill here, and must override this: the default seats only a strategy that carries no per-row state between steps.
-
inline virtual bool prefillSeatedChunk(DecodingInferenceContext&)#
Run the draft prefill over a non-final chunk of a sequence being seated (the stable prefix before a Hybrid+spec volatile tail), in the same projection as seatRow(). seatRow() follows after the final chunk.
- inline virtual std::vector<int32_t> const &commonMaterializedStateLengths(
Greatest per-slot logical prefix whose continuation state is materialized by every model in this strategy. Physical model-state tails may extend beyond this boundary. This reports decoding progress only; context-cache policy decides whether that prefix can be published.
-
virtual int64_t getRequiredContextMemorySize() const noexcept = 0#
-
virtual ~DecodingStrategy() noexcept = default#
-
struct DecodingStrategyCapabilities#
Public Members
-
bool ownsBaseVerificationCudaGraphs = {false}#
-
bool supportsLosslessSampling = {false}#
-
int32_t maxSamplingSupport = {0}#
0 when the decoder does not require a bounded sampling support.
-
bool fallbackToVanillaForNonGreedySampling = {false}#
Preserve request sampling semantics instead of coercing a greedy-only strategy.
-
bool requiresDefaultDecoderCudaGraphs = {false}#
Capture vanilla graphs when request routing may select the default decoder.
-
bool ownsBaseVerificationCudaGraphs = {false}#
-
struct DecodingKvHeadroomPolicy#
KV headroom a strategy declares for its decode rounds.
Public Members
-
DecodingKvHeadroom plain#
every slot, every step
-
DecodingKvHeadroom jumpForward#
a slot that runs with jump-forward; equal to plain when rounds are not widened
-
DecodingKvHeadroom plain#
-
struct SamplingBuffers#
Public Members
-
Tensor &hostOutputSpaceIds#
Sampled indices captured before mapReducedVocabToFullVocab, i.e. still in the engine’s output vocabulary. Grammar matchers live in that space (see GuidedDecoder), so they must be advanced with these rather than the remapped full IDs.
-
Tensor &rows#
One SamplingRow per active row for the step (parameters and uniform), GPU bytes and the pinned host staging it is copied from.
-
Tensor &hostOutputSpaceIds#
-
struct LogprobsBuffers#
GPU/CPU buffers for logprobs computation (owned by LLMInferenceRuntime). Referenced here so decoders can call log-softmax + top-K on the output logits.
Public Members
-
struct BaseEngineResources#
Base-engine execution infrastructure: executor, tensor map, KV cache, pipeline I/O, shared resources, and CUDA-graph capture callback.
Public Members
-
EngineExecutor &executor#
-
HybridCacheManager &cacheManager#
-
PipelineIO &pipelineIO#
-
std::function<bool(InferenceDims const&, cudaStream_t)> captureGraph#
-
EngineExecutor &executor#
-
struct PreprocessResources#
Preprocessing resources: embedding lookup, step preparation, deepstack.
Public Members
-
StepPreparer &stepPreparer#
-
EmbeddingPreprocessor &embeddingPreprocessor#
-
EmbeddingData &embedding#
-
DeepstackBinding *deepstack#
-
Gemma4EmbeddingPreprocessor *gemma4Ple#
-
Qwen4ExpNgramPreprocessor *qwen4ExpNgram#
-
StepPreparer &stepPreparer#
-
struct DecodingRuntimeContext#
Public Members
-
DeploymentConfig &deployment#
-
int32_t maxRuntimeBatchSize#
-
std::filesystem::path const &checkpointDir#
-
std::filesystem::path const &draftCheckpointDir#
-
BaseEngineResources base#
-
PreprocessResources preprocess#
-
GuidedDecoder &guidedDecoder#
-
SamplingBuffers sampling#
-
LogprobsBuffers logprobs#
-
std::function<bool(void *buffer, int32_t count, cudaStream_t stream)> tokenBroadcast = {}#
Optional sampled-token synchronization callback. Rank 0 sends; peer ranks receive.
-
int32_t parallelRank = {-1}#
-
RuntimeBenchmarkOptions const *benchmark = {}#
-
SpecPleStaging specPleStaging = {}#
PLE inputs of speculative target verify; set up once the base engine is loaded.
-
int32_t maxJumpTokens = {0}#
Tokens one jump may add in the speculative decoder (SpecDecodeConfig::maxJumpTokens); 0 disables its jumps.
-
int32_t chunkMaxJumpTokens = {0}#
Tokens one jump may add in the vanilla decoder, a speculative deployment’s fallback included: what one catch-up context-chunk row holds besides the sampled token; 0 disables its jumps.
-
DeploymentConfig &deployment#