EAGLE Decoder

class EagleDecoder : public trt_edgellm::rt::DecodingStrategy

Public Functions

EagleDecoder(

DecodingRuntimeContext &runtime,

std::filesystem::path const &engineDir,

SpecDecodeDraftingConfig const &draftingConfig,

std::unique_ptr<EngineExecutor> draftExecutor,

cudaStream_t stream

)

inline virtual DecodingStrategyKind kind() const noexcept override

inline virtual char const *name() const noexcept override

inline virtual bool isSpeculative() const noexcept override

virtual bool decodeStep(DecodingInferenceContext &context) override

virtual bool captureCudaGraphs(cudaStream_t stream) override

virtual bool initializeForGeneration(

DecodingInferenceContext &context

) override

Initialize decoder-private generation state after base prefill. Non-speculative strategies are no-ops.

virtual std::vector<int32_t> const &commonMaterializedStateLengths(

) const noexcept override

Greatest per-slot logical prefix whose continuation state is materialized by every model in this strategy. Physical model-state tails may extend beyond this boundary. This reports decoding progress only; context-cache policy decides whether that prefix can be published.

virtual int64_t getRequiredContextMemorySize(

) const noexcept override

virtual void setContextMemory(Tensor &memory) override

virtual bool hasSystemPromptKVCache(

SystemPromptCacheKey const &key

) const override

virtual void restoreSystemPromptKVCache(

SystemPromptCacheKey const &key,

int32_t batchIdx,

cudaStream_t stream

) override

virtual bool runSystemPromptPrefill(

DecodingInferenceContext &context

) override

virtual void saveSystemPromptKVCache(

SystemPromptCacheKey const &key,

std::string const &prompt,

std::vector<tokenizer::Rank> const &tokenizedPrompt,

int32_t promptIdsLength,

cudaStream_t stream

) override

virtual void resetForNewSequences(

Tensor &reuseLengths,

cudaStream_t stream

) override

virtual void onBatchEvict(

std::vector<int32_t> const &batchMapping,

int32_t oldActiveBatch,

int32_t newActiveBatch,

Tensor &deviceBatchMapping,

cudaStream_t stream,

BatchCompactionMode mode

) override