Dflash Decode Utils

struct CachedBlockDraftRuntimeConfig

Public Members

SpecDecodeMode userMode = {SpecDecodeMode::kNONE}

ProposalAttentionPolicy proposalAttention = {ProposalAttentionPolicy::kBidirectional}

BlockDraftTreePolicy treePolicy = {BlockDraftTreePolicy::kLinear}

int32_t blockSize = {0}

int32_t proposalLen = {0}

int32_t verifySize = {0}

int32_t candidateTopK = {1}

int32_t maskTokenId = {0}

int32_t draftHiddenSize = {0}

int32_t baseOutputHiddenDim = {0}

int32_t draftVocabSize = {0}

char const *trt_edgellm::rt::dflash_utils::proposalAttentionPolicyName(

ProposalAttentionPolicy policy

) noexcept

char const *trt_edgellm::rt::dflash_utils::blockDraftTreePolicyName(

BlockDraftTreePolicy policy

) noexcept

int32_t trt_edgellm::rt::dflash_utils::runtimeBlockSize(

DeploymentConfig const &deployment

)

Return the DFlash/JetSpec draft block horizon used by runtime draft forward.

DeploymentConfig resolves explicit/user-supplied DFlash/JetSpec horizons before the decoder is created, so the runtime reads one consolidated value.

bool trt_edgellm::rt::dflash_utils::shouldUseDDTree(

DeploymentConfig const &deployment

)

DFlash/JetSpec uses a linear-tree proposal when draftingTopK == 1 and a branching DDTree proposal when draftingTopK > 1.

CachedBlockDraftRuntimeConfig trt_edgellm::rt::dflash_utils::makeCachedBlockDraftRuntimeConfig(

DeploymentConfig const &deployment

)