Action KV Batch#

class ActionKvBatchCollector#

Collects request-local terminal KV rows and materializes the final action-local paged batch.

Public Functions

ActionKvBatchCollector(
int32_t maxBatch,
int32_t maxPagesPerSeq,
int32_t numPages,
ContextCacheManager *pageManager = nullptr
)#
void beginRequest(
std::vector<bool> const &actionSlots,
std::vector<int64_t> const &mropeDeltas
)#
void captureFinished(
KVPageTable const &pageTable,
Tensor const &deviceKvLengths,
std::vector<int8_t> const &finished,
std::vector<int32_t> const &originalIndices,
std::vector<ResidentRef> const &residentRefs,
cudaStream_t stream
)#
void completeCapture()#

Complete pending snapshots after length transfers finish. Managed snapshots complete inside captureFinished.

ActionKvBatchView materialize(cudaStream_t stream)#

Validate completeness and materialize one action-local logical KV batch.

void releaseMaterialized(cudaStream_t stream)#

Release retained prefix pins after all dependent GPU work on stream is terminal.

std::vector<int32_t> const &originalRequestIndices(
) const noexcept#
struct ActionKvBatchView#

Retained read-only LLM prefixes, compacted into action request order.

Public Members

KVPageTable const &pageTable#
Tensor const &kvLengthsDevice#
Tensor const &kvLengthsHost#
std::vector<int64_t> const &mropeDeltas#
int32_t batchSize = {}#