Cute Dsl FMHA Params#
-
struct LlmFmhaParams#
Everything the dense LLM descriptors need, gathered once per CuteDslFMHARunner::run() call.
Public Members
-
void const *qPtr = {}#
-
void const *kvPtr = {}#
-
void *oPtr = {}#
-
int32_t const *cuKVSeqLens = {}#
-
int32_t batchSize = {}#
-
int32_t seqLenQ = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t kvCacheCapacity = {}#
-
int32_t windowSizeLeft = {}#
-
float attentionScale = {}#
-
float scaleQ = {}#
-
float scaleK = {}#
-
float scaleV = {}#
-
float invScaleO = {}#
-
int32_t const *skipSoftmaxScaleTable = {}#
Skip-softmax (BLASST) scale table, device Int32[skipSoftmaxScaleTableLen] of flattened (seqlen_bin, S) pairs; nullptr / 0 = disabled. Only the *_skipsoftmax launchers forward it.
-
int32_t skipSoftmaxScaleTableLen = {}#
-
int32_t const *skipSoftmaxCuKVSeqLens = {}#
Actual (unpadded) cumulative KV lengths [batchSize + 1] for the skip-softmax lookup; cuKVSeqLens may be padded to the batch’s longest query (bottom-right alignment). Skip variants only.
-
cudaStream_t stream = {}#
-
void const *qPtr = {}#
-
struct LlmFmhaPagedParams#
Everything the paged LLM descriptors need, gathered once per CuteDslFMHARunner::runPaged() call.
Public Members
-
void const *actionPoolPtr = {}#
-
int32_t const *actionPageList = {}#
-
int32_t numActionFlatPages = {}#
-
int32_t maxActionPagesPerSeq = {}#
-
void const *qPtr = {}#
-
void const *pagedKVPoolPtr = {}#
-
int32_t const *kvCachePageList = {}#
-
void *oPtr = {}#
-
int32_t const *cuQSeqLens = {}#
-
int32_t const *cuKVSeqLens = {}#
-
int32_t const *bidirectionalBlockBegin = {}#
-
int32_t const *bidirectionalBlockEnd = {}#
-
int32_t batchSize = {}#
-
int32_t seqLenQ = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t numPages = {}#
-
int32_t maxPagesPerSeq = {}#
-
int32_t tokensPerPage = {}#
-
int32_t windowSizeLeft = {}#
-
float attentionScale = {}#
-
float scaleQ = {}#
-
float scaleK = {}#
-
float scaleV = {}#
-
float invScaleO = {}#
-
int32_t const *skipSoftmaxScaleTable = {}#
Skip-softmax (BLASST) scale table, device Int32[skipSoftmaxScaleTableLen] of flattened (seqlen_bin, S) pairs; nullptr / 0 = disabled. Forwarded by the optimized *_skipsoftmax_paged launchers and by every FMHA-v2 causal paged launcher (whose kernels carry the path; the FMHA-v2 runner substitutes the dense {0, 0} row when unset).
-
int32_t skipSoftmaxScaleTableLen = {}#
-
int32_t const *skipSoftmaxCuKVSeqLens = {}#
Actual (unpadded) cumulative KV lengths [batchSize + 1] for the skip-softmax lookup; cuKVSeqLens may be padded to the batch’s longest query (bottom-right alignment). Optimized skip variants only — FMHA-v2’s cuKVSeqLens is never padded, so its kernel looks the table up by cuKVSeqLens directly.
-
cudaStream_t stream = {}#
-
void const *actionPoolPtr = {}#
-
struct LlmFmhaRaggedPagedParams#
Everything the optimized Blackwell packed-Q/O paged descriptors need for one ragged launch.
Public Members
-
int32_t const *bidirectionalBlockBegin = {}#
Packed [totalQSeqLen] sequence-relative inclusive KV positions.
-
int32_t const *bidirectionalBlockEnd = {}#
Paired with begin; text rows use -1 for both boundaries.
-
float const *attentionSinks = {}#
Optional device FP32 logits [numQHeads].
-
void const *qPtr = {}#
-
void const *pagedKVPoolPtr = {}#
-
int32_t const *kvCachePageList = {}#
-
void *oPtr = {}#
-
int32_t const *cuQSeqLens = {}#
-
int32_t const *cuKVSeqLens = {}#
-
int32_t totalQSeqLen = {}#
-
int32_t maxQSeqLen = {}#
-
int32_t batchSize = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t numFlatPages = {}#
-
int32_t maxPagesPerSeq = {}#
-
int32_t tokensPerPage = {}#
-
int32_t windowSizeLeft = {}#
-
float attentionScale = {}#
-
float scaleQ = {}#
-
float scaleK = {}#
-
float scaleV = {}#
-
float invScaleO = {}#
-
cudaStream_t stream = {}#
-
int32_t const *bidirectionalBlockBegin = {}#
-
struct VitFmhaParams#
Everything the ViT descriptors need, gathered once per CuteDslFMHARunner ViT run() call. The ViT AOT variants are plain MHA, hence a single head count.
Public Members
-
void const *qPtr = {}#
-
void const *kPtr = {}#
-
void const *vPtr = {}#
-
void *oPtr = {}#
-
int32_t const *cuSeqLens = {}#
-
int32_t totalSeqLen = {}#
-
int32_t numHeads = {}#
-
int32_t headDim = {}#
-
int32_t maxSeqLen = {}#
-
int32_t batchSize = {}#
-
float scaleSoftmaxLog2 = {}#
-
float attentionScale = {}#
-
float scaleOutput = {}#
-
cudaStream_t stream = {}#
-
void const *qPtr = {}#
-
struct FmhaV2LlmParams#
Everything the FMHA-v2 dense LLM descriptors need, gathered once per CuteDslFMHAV2Runner::run() call.
Public Members
-
void const *qPtr = {}#
-
void const *kPtr = {}#
-
void const *vPtr = {}#
-
void *oPtr = {}#
-
int32_t const *cuKVSeqLens = {}#
-
int32_t batchSize = {}#
-
int32_t seqLenQ = {}#
-
int32_t kvSeqLen = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t windowSizeLeft = {}#
-
float attentionScale = {}#
-
float scaleQ = {}#
-
float scaleK = {}#
-
float scaleV = {}#
-
float invScaleO = {}#
-
cudaStream_t stream = {}#
-
int32_t const *bidirectionalBlockBegin = {}#
Per-query-row [begin, end) block intervals, packed [B, S_q]. Both null for the plain causal/sliding variants; both set for the bidirectional variants.
-
int32_t const *bidirectionalBlockEnd = {}#
-
void const *qPtr = {}#
-
struct FmhaV2RaggedPagedParams#
Everything the packed-Q/O FMHA-v2 paged descriptors need, gathered once per ragged launch.
Public Members
-
void const *qPtr = {}#
-
void const *pagedKVPoolPtr = {}#
-
int32_t const *kvCachePageList = {}#
-
void *oPtr = {}#
-
int32_t const *cuQSeqLens = {}#
-
int32_t const *cuKVSeqLens = {}#
-
int32_t const *bidirectionalBlockBegin = {}#
-
int32_t const *bidirectionalBlockEnd = {}#
-
int32_t totalQSeqLen = {}#
-
int32_t maxQSeqLen = {}#
-
int32_t batchSize = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t numPages = {}#
-
int32_t maxPagesPerSeq = {}#
-
int32_t tokensPerPage = {}#
-
int32_t windowSizeLeft = {}#
-
float attentionScale = {}#
-
float const *attentionSinks = {}#
-
cudaStream_t stream = {}#
-
void const *qPtr = {}#
-
struct FmhaV2VitParams#
Everything the FMHA-v2 ViT descriptors need, gathered once per CuteDslFMHAV2Runner ViT run() call. Unlike VitFmhaParams these variants are GQA, so Q and KV head counts differ.
Public Members
-
void const *qPtr = {}#
-
void const *kPtr = {}#
-
void const *vPtr = {}#
-
void *oPtr = {}#
-
int32_t const *cuSeqLens = {}#
-
int32_t totalSeqLen = {}#
-
int32_t numQHeads = {}#
-
int32_t numKVHeads = {}#
-
int32_t headDim = {}#
-
int32_t maxSeqLen = {}#
-
int32_t batchSize = {}#
-
float scaleSoftmaxLog2 = {}#
-
float attentionScale = {}#
-
float scaleOutput = {}#
-
cudaStream_t stream = {}#
-
void const *qPtr = {}#