Decoder Registry

class DecoderRegistry

Public Functions

DecoderRegistry(

DecodingRuntimeContext &runtime,

DecoderRegistryInit init

)

DecodingStrategy &select(

LLMGenerationRequest const &request

) const noexcept

DecodingStrategy &cachePrimingStrategy() const noexcept

bool captureCudaGraphs(cudaStream_t stream) const

int64_t getRequiredContextMemorySize() const noexcept

void setContextMemory(Tensor &memory) const

inline bool hasSpeculativeDecoder() const noexcept

inline char const *speculativeDecoderName() const noexcept

struct DecoderRegistryInit

Public Members

std::filesystem::path engineDir

std::optional<SpecDecodeDraftingConfig> draftingConfig

std::unique_ptr<EngineExecutor> draftExecutor

Validated draft executor whose ownership is consumed by the selected speculative decoder.

cudaStream_t stream = {}

bool trt_edgellm::rt::shouldSelectDefaultDecoder(

DecodingStrategyKind speculativeDecoderKind,

LLMGenerationRequest const &request

) noexcept

Return whether an available speculative decoder should yield to the default decoder for this request.