Cute Dsl FMHA Params#

struct LlmFmhaParams#

Everything the dense LLM descriptors need, gathered once per CuteDslFMHARunner::run() call.

Public Members

void const *qPtr = {}#
void const *kvPtr = {}#
void *oPtr = {}#
int32_t const *cuKVSeqLens = {}#
int32_t batchSize = {}#
int32_t seqLenQ = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t kvCacheCapacity = {}#
int32_t windowSizeLeft = {}#
float attentionScale = {}#
float scaleQ = {}#
float scaleK = {}#
float scaleV = {}#
float invScaleO = {}#
int32_t const *skipSoftmaxScaleTable = {}#

Skip-softmax (BLASST) scale table, device Int32[skipSoftmaxScaleTableLen] of flattened (seqlen_bin, S) pairs; nullptr / 0 = disabled. Only the *_skipsoftmax launchers forward it.

int32_t skipSoftmaxScaleTableLen = {}#
int32_t const *skipSoftmaxCuKVSeqLens = {}#

Actual (unpadded) cumulative KV lengths [batchSize + 1] for the skip-softmax lookup; cuKVSeqLens may be padded to the batch’s longest query (bottom-right alignment). Skip variants only.

cudaStream_t stream = {}#
struct LlmFmhaPagedParams#

Everything the paged LLM descriptors need, gathered once per CuteDslFMHARunner::runPaged() call.

Public Members

void const *actionPoolPtr = {}#
int32_t const *actionPageList = {}#
int32_t numActionFlatPages = {}#
int32_t maxActionPagesPerSeq = {}#
void const *qPtr = {}#
void const *pagedKVPoolPtr = {}#
int32_t const *kvCachePageList = {}#
void *oPtr = {}#
int32_t const *cuQSeqLens = {}#
int32_t const *cuKVSeqLens = {}#
int32_t const *bidirectionalBlockBegin = {}#
int32_t const *bidirectionalBlockEnd = {}#
int32_t batchSize = {}#
int32_t seqLenQ = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t numPages = {}#
int32_t maxPagesPerSeq = {}#
int32_t tokensPerPage = {}#
int32_t windowSizeLeft = {}#
float attentionScale = {}#
float scaleQ = {}#
float scaleK = {}#
float scaleV = {}#
float invScaleO = {}#
int32_t const *skipSoftmaxScaleTable = {}#

Skip-softmax (BLASST) scale table, device Int32[skipSoftmaxScaleTableLen] of flattened (seqlen_bin, S) pairs; nullptr / 0 = disabled. Forwarded by the optimized *_skipsoftmax_paged launchers and by every FMHA-v2 causal paged launcher (whose kernels carry the path; the FMHA-v2 runner substitutes the dense {0, 0} row when unset).

int32_t skipSoftmaxScaleTableLen = {}#
int32_t const *skipSoftmaxCuKVSeqLens = {}#

Actual (unpadded) cumulative KV lengths [batchSize + 1] for the skip-softmax lookup; cuKVSeqLens may be padded to the batch’s longest query (bottom-right alignment). Optimized skip variants only — FMHA-v2’s cuKVSeqLens is never padded, so its kernel looks the table up by cuKVSeqLens directly.

cudaStream_t stream = {}#
struct LlmFmhaRaggedPagedParams#

Everything the optimized Blackwell packed-Q/O paged descriptors need for one ragged launch.

Public Members

int32_t const *bidirectionalBlockBegin = {}#

Packed [totalQSeqLen] sequence-relative inclusive KV positions.

int32_t const *bidirectionalBlockEnd = {}#

Paired with begin; text rows use -1 for both boundaries.

float const *attentionSinks = {}#

Optional device FP32 logits [numQHeads].

void const *qPtr = {}#
void const *pagedKVPoolPtr = {}#
int32_t const *kvCachePageList = {}#
void *oPtr = {}#
int32_t const *cuQSeqLens = {}#
int32_t const *cuKVSeqLens = {}#
int32_t totalQSeqLen = {}#
int32_t maxQSeqLen = {}#
int32_t batchSize = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t numFlatPages = {}#
int32_t maxPagesPerSeq = {}#
int32_t tokensPerPage = {}#
int32_t windowSizeLeft = {}#
float attentionScale = {}#
float scaleQ = {}#
float scaleK = {}#
float scaleV = {}#
float invScaleO = {}#
cudaStream_t stream = {}#
struct VitFmhaParams#

Everything the ViT descriptors need, gathered once per CuteDslFMHARunner ViT run() call. The ViT AOT variants are plain MHA, hence a single head count.

Public Members

void const *qPtr = {}#
void const *kPtr = {}#
void const *vPtr = {}#
void *oPtr = {}#
int32_t const *cuSeqLens = {}#
int32_t totalSeqLen = {}#
int32_t numHeads = {}#
int32_t headDim = {}#
int32_t maxSeqLen = {}#
int32_t batchSize = {}#
float scaleSoftmaxLog2 = {}#
float attentionScale = {}#
float scaleOutput = {}#
cudaStream_t stream = {}#
struct FmhaV2LlmParams#

Everything the FMHA-v2 dense LLM descriptors need, gathered once per CuteDslFMHAV2Runner::run() call.

Public Members

void const *qPtr = {}#
void const *kPtr = {}#
void const *vPtr = {}#
void *oPtr = {}#
int32_t const *cuKVSeqLens = {}#
int32_t batchSize = {}#
int32_t seqLenQ = {}#
int32_t kvSeqLen = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t windowSizeLeft = {}#
float attentionScale = {}#
float scaleQ = {}#
float scaleK = {}#
float scaleV = {}#
float invScaleO = {}#
cudaStream_t stream = {}#
int32_t const *bidirectionalBlockBegin = {}#

Per-query-row [begin, end) block intervals, packed [B, S_q]. Both null for the plain causal/sliding variants; both set for the bidirectional variants.

int32_t const *bidirectionalBlockEnd = {}#
struct FmhaV2RaggedPagedParams#

Everything the packed-Q/O FMHA-v2 paged descriptors need, gathered once per ragged launch.

Public Members

void const *qPtr = {}#
void const *pagedKVPoolPtr = {}#
int32_t const *kvCachePageList = {}#
void *oPtr = {}#
int32_t const *cuQSeqLens = {}#
int32_t const *cuKVSeqLens = {}#
int32_t const *bidirectionalBlockBegin = {}#
int32_t const *bidirectionalBlockEnd = {}#
int32_t totalQSeqLen = {}#
int32_t maxQSeqLen = {}#
int32_t batchSize = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t numPages = {}#
int32_t maxPagesPerSeq = {}#
int32_t tokensPerPage = {}#
int32_t windowSizeLeft = {}#
float attentionScale = {}#
float const *attentionSinks = {}#
cudaStream_t stream = {}#
struct FmhaV2VitParams#

Everything the FMHA-v2 ViT descriptors need, gathered once per CuteDslFMHAV2Runner ViT run() call. Unlike VitFmhaParams these variants are GQA, so Q and KV head counts differ.

Public Members

void const *qPtr = {}#
void const *kPtr = {}#
void const *vPtr = {}#
void *oPtr = {}#
int32_t const *cuSeqLens = {}#
int32_t totalSeqLen = {}#
int32_t numQHeads = {}#
int32_t numKVHeads = {}#
int32_t headDim = {}#
int32_t maxSeqLen = {}#
int32_t batchSize = {}#
float scaleSoftmaxLog2 = {}#
float attentionScale = {}#
float scaleOutput = {}#
cudaStream_t stream = {}#