Cute Dsl Fmhav2 Runner#

class CuteDslFMHAV2Runner#

Runner for the CuTe DSL FMHA-v2 kernels.

Dense LLM kernels consume separate BSND Q/K/V tensors, while native-paged LLM kernels consume BSND or packed Q/O and the Edge-LLM NHD paged KV pool. Vision blocks support both dense separate Q/K/V and packed Q/O with paged KV; padding uses dense Q/K/V and ViT uses packed separate Q/K/V. The runner remains separate from CuteDslFMHARunner because the optimized SM100/101/103/110 family has a distinct ABI.

Public Functions

CuteDslFMHAV2Runner(
int32_t numQHeads,
int32_t numKVHeads,
int32_t headDim,
int32_t batchSize = 0,
int32_t seqLenQ = 0,
int32_t kvSeqLen = 0,
bool useSmallD64 = true
)#
~CuteDslFMHAV2Runner() = default#
CuteDslFMHAV2Runner(CuteDslFMHAV2Runner const&) = delete#
CuteDslFMHAV2Runner &operator=(CuteDslFMHAV2Runner const&) = delete#
bool preflightLlm(
cudaStream_t stream,
int32_t slidingWindowSize = INT_MAX
)#

Ensures the exact causal or sliding-causal LLM variant selected by run() is loaded.

bool preflightPaged(
cudaStream_t stream,
int32_t slidingWindowSize = INT_MAX,
int32_t const *skipSoftmaxScaleTable = nullptr,
int32_t skipSoftmaxScaleTableLen = 0
)#

Ensures the exact native-paged causal or sliding-causal LLM variant selected by runPaged() is loaded (the scale table does not select a variant).

bool preflightPagedRagged(
cudaStream_t stream,
int32_t slidingWindowSize = INT_MAX,
bool useBidirectional = false
)#

Loads the causal, sliding-causal, or vision-block variant selected by runPagedRagged().

bool preflightPadding(cudaStream_t stream)#

Ensures the dense non-causal padding variant selected by runPadding() is loaded.

bool preflightVisionBlock(cudaStream_t stream)#

Ensures the vision-block variant selected by runVisionBlock() is loaded.

bool preflightViT(cudaStream_t stream)#

Ensures the exact packed ViT variant selected by run() is loaded.

bool run(
void const *qPtr,
void const *kPtr,
void const *vPtr,
void *oPtr,
int32_t const *cuKVSeqLens,
cudaStream_t stream,
float attentionScale,
int32_t slidingWindowSize = INT_MAX
)#

Runs causal or sliding-causal LLM context attention over dense FP16 K/V.

bool runPaged(
void const *qPtr,
void const *pagedKVPoolPtr,
int32_t const *kvCachePageList,
void *oPtr,
int32_t const *cuQSeqLens,
int32_t const *cuKVSeqLens,
int32_t numFlatPages,
int32_t maxPagesPerSeq,
int32_t tokensPerPage,
cudaStream_t stream,
float attentionScale,
int32_t slidingWindowSize = INT_MAX,
int32_t const *skipSoftmaxScaleTable = nullptr,
int32_t skipSoftmaxScaleTableLen = 0
)#

Runs FP16 causal or sliding-causal attention directly against an FP16 NHD paged KV pool.

Parameters:
  • skipSoftmaxScaleTable – Skip-softmax (BLASST) scale table, device Int32 of flattened (seqlen_bin, S) pairs with bins ascending; a flat S is the one-row table {0, S}. Causal, non-sliding only: per sequence the kernel floor-looks-up S by its KV length (cuKVSeqLens, which is never padded here) and skips the P*V GEMM of KV tiles whose contribution is below lambda = S / seqlen_kv of the running max — approximate. Sequences below the first bin or with S >= seqlen_kv run dense.

  • skipSoftmaxScaleTableLen – Element count of the table; 0 (the default) runs every sequence dense.

bool preflightPagedTwoPoolNonCausal(
cudaStream_t stream,
bool fp8Prefix
)#

Load the separate-prefix variant before CUDA graph capture.

bool runPagedTwoPoolNonCausal(
void const *qPtr,
void const *prefixPool,
int32_t const *prefixPages,
void const *actionPool,
int32_t const *actionPages,
void *oPtr,
int32_t const *cuQSeqLens,
int32_t const *cuKVSeqLens,
int32_t numPrefixFlatPages,
int32_t maxPrefixPagesPerSeq,
int32_t numActionFlatPages,
int32_t maxActionPagesPerSeq,
cudaStream_t stream,
float attentionScale,
bool fp8Prefix,
float kScale,
float vScale
)#

Read a retained prefix and independent FP16 action pages with one non-causal softmax.

bool runPagedRagged(PagedRaggedParams const &inputs)#

Runs causal, sliding-causal, or vision-block attention over packed Q/O and FP16 NHD paged KV.

cuQSeqLens and cuKVSeqLens are monotonic prefix arrays of length batchSize + 1. cuQSeqLens starts at zero and ends at totalQSeqLen. cuKVSeqLens contains the complete logical KV history, including prior cached tokens. maxQSeqLen is the maximum adjacent difference in cuQSeqLens, not the packed buffer allocation capacity. attentionSinks optionally points to device-resident FP32 sink logits indexed by query head; nullptr disables sinks. bidirectionalBlockBegin and bidirectionalBlockEnd optionally contain inclusive, sequence-relative KV intervals, indexed by packed query row [totalQSeqLen]. Text rows use the empty interval [-1, -2]. Vision blocks must be contiguous and contain the query at KV position (kvLength - qLength + queryRow).

bool runPadding(
void const *qPtr,
void const *kPtr,
void const *vPtr,
void *oPtr,
int32_t const *cuQSeqLens,
int32_t const *cuKVSeqLens,
cudaStream_t stream,
float attentionScale,
int32_t const *bidirectionalBlockBegin = nullptr,
int32_t const *bidirectionalBlockEnd = nullptr
)#

Runs dense non-causal padded context attention with independent logical Q/KV lengths. D512 additionally needs [B, S_q] full-KV intervals from launchBuildPaddingBlockRanges().

bool runVisionBlock(
void const *qPtr,
void const *kPtr,
void const *vPtr,
void *oPtr,
int32_t const *cuKVSeqLens,
int32_t const *bidirectionalBlockBegin,
int32_t const *bidirectionalBlockEnd,
cudaStream_t stream,
float attentionScale,
int32_t slidingWindowSize
)#

Runs Gemma4 vision-block attention: sliding-causal OR same-image-block.

bool run(
void const *qPtr,
void const *kPtr,
void const *vPtr,
void *oPtr,
int32_t const *cuSeqLens,
int32_t totalSeqLen,
int32_t maxSeqLen,
int32_t batchSize,
cudaStream_t stream,
float attentionScale
)#

Runs packed varlen, bidirectional ViT attention.

Public Static Functions

static bool canImplement(
int32_t numQHeads,
int32_t numKVHeads,
int32_t headSize,
int32_t smVersion,
trt_edgellm::DataType dataType,
CuteDslFMHAV2MaskType maskType
)#

Returns whether the target AOT family covers this dense context shape.

static bool canImplementPaged(
int32_t numQHeads,
int32_t numKVHeads,
int32_t headSize,
int32_t smVersion,
trt_edgellm::DataType dataType,
CuteDslFMHAV2MaskType maskType
)#

Returns whether the target AOT family covers native FP16 Edge-LLM paged KV for this context shape.

static bool canImplementPagedRagged(
int32_t numQHeads,
int32_t numKVHeads,
int32_t headSize,
int32_t smVersion,
trt_edgellm::DataType dataType,
CuteDslFMHAV2MaskType maskType
)#

Returns whether the target AOT family covers packed-Q/O native paged attention.

static bool canImplementViT(
int32_t headSize,
int32_t smVersion,
trt_edgellm::DataType dataType
)#

Returns whether the target AOT family covers this packed ViT shape.

struct PagedRaggedParams#

Inputs for packed Q/O attention over the native NHD paged KV cache.

Public Members

void const *qPtr = {}#
void const *pagedKVPoolPtr = {}#
int32_t const *kvCachePageList = {}#
void *oPtr = {}#
int32_t const *cuQSeqLens = {}#
int32_t const *cuKVSeqLens = {}#
int32_t totalQSeqLen = {}#
int32_t maxQSeqLen = {}#
int32_t numFlatPages = {}#
int32_t maxPagesPerSeq = {}#
int32_t tokensPerPage = {}#
cudaStream_t stream = {}#
float attentionScale = {}#
int32_t slidingWindowSize = {INT_MAX}#
float const *attentionSinks = {}#
int32_t const *bidirectionalBlockBegin = {}#
int32_t const *bidirectionalBlockEnd = {}#
struct PagedRaggedParams

Inputs for packed Q/O attention over the native NHD paged KV cache.

Public Members

void const *qPtr = {}
void const *pagedKVPoolPtr = {}
int32_t const *kvCachePageList = {}
void *oPtr = {}
int32_t const *cuQSeqLens = {}
int32_t const *cuKVSeqLens = {}
int32_t totalQSeqLen = {}
int32_t maxQSeqLen = {}
int32_t numFlatPages = {}
int32_t maxPagesPerSeq = {}
int32_t tokensPerPage = {}
cudaStream_t stream = {}
float attentionScale = {}
int32_t slidingWindowSize = {INT_MAX}
float const *attentionSinks = {}
int32_t const *bidirectionalBlockBegin = {}
int32_t const *bidirectionalBlockEnd = {}