Bounded Swa KV Page Manager#

class BoundedSwaKVPageManager#

Runtime-level allocator and execution-lifecycle owner for bounded SWA pages.

KVCacheManager owns the device buffers. This class owns only page IDs, sparse-table mappings, and request-local transition state. Pages staged for retirement remain leased until the existing CUDA completion point.

Public Types

using StreamSynchronizer = std::function<cudaError_t(cudaStream_t)>#

Public Functions

BoundedSwaKVPageManager(
HybridCacheManager &baseCache,
KVPageTable &basePageTable,
KVPageTable &swaPageTable,
cudaStream_t stream,
StreamSynchronizer synchronizer = {},
bool externalPageOwnership = false
)#
~BoundedSwaKVPageManager() noexcept#
BoundedSwaKVPageManager(BoundedSwaKVPageManager const&) = delete#
BoundedSwaKVPageManager &operator=(
BoundedSwaKVPageManager const&
) = delete#
BeginRequestResult beginRequest(
std::vector<ResidentRef> const &residents,
cudaStream_t stream
)#
BeginRequestResult beginRequest(
std::vector<SwaKVCacheState> states,
cudaStream_t stream
)#

Begin a request from pages already pinned atomically by the context-cache lease.

SwaKVCacheStatus preparePrefill(
RequestHandle &request,
std::vector<int32_t> const &inputLengths,
ExternalPageLifecycle const &lifecycle = {}
)#
SwaKVCacheStatus completePrefill(
RequestHandle &request,
ExternalPageLifecycle const &lifecycle = {}
)#
SwaKVCacheStatus prepareDecodeStep(
RequestHandle &request,
ExternalPageLifecycle const &lifecycle = {},
std::vector<int32_t> const *advanceCounts = nullptr
)#
Parameters:

advanceCounts – Tokens each slot’s decode step writes, in slot order; null means one per slot.

SwaKVCacheStatus completeDecodeStep(
RequestHandle &request,
ExternalPageLifecycle const &lifecycle = {}
)#
SwaKVCacheStatus beginBatchCompaction(
RequestHandle &request,
std::vector<int32_t> const &oldToNew,
int32_t newBatchSize,
Tensor &deviceBatchMapping
)#
SwaKVCacheStatus compactBatch(RequestHandle &request)#
SwaKVCacheStatus finish(RequestHandle &request)#
SwaKVCacheStatus shutdown() noexcept#
int32_t freePageCount() const noexcept#
int32_t privatePagesPerSequence() const noexcept#
SwaKVCacheState const &state(
RequestHandle const &request,
int32_t slot
) const#
SwaWindowStateView states(RequestHandle const &request) const#
struct BeginRequestResult#

Public Members

SwaKVCacheStatus status = {SwaKVCacheStatus::kRequestFailed}#
std::optional<RequestHandle> request#
struct ExternalPageLifecycle#

Hooks used only when a context-cache lease owns the request’s page references.

Public Members

std::function<bool(int32_t, PageId)> isExternallyOwned#
std::function<PageId(int32_t, PageId)> replace#
class RequestHandle#

Public Functions

RequestHandle(RequestHandle &&other) noexcept#
RequestHandle &operator=(RequestHandle &&other) = delete#
RequestHandle(RequestHandle const&) = delete#
RequestHandle &operator=(RequestHandle const&) = delete#
~RequestHandle() noexcept#
bool valid() const noexcept#
struct Impl#

Public Types

enum class Phase : uint8_t#

Values:

enumerator kAdmitted#
enumerator kExecuting#
enumerator kFinishing#

Public Functions

inline Impl(BoundedSwaKVPageManager &manager_, cudaStream_t stream_)#

Public Members

BoundedSwaKVPageManager *manager = {}#
cudaStream_t stream = {}#
std::vector<int32_t> inputLengths#
std::vector<SwaKVCacheState> states#
std::vector<int32_t> pendingCompactionMapping#
int32_t pendingCompactionBatchSize = {-1}#
Tensor const *pendingDeviceBatchMapping = {}#
Phase phase = {Phase::kAdmitted}#
bool deviceWorkPending = {}#
class RequestHandle

Public Functions

RequestHandle(RequestHandle &&other) noexcept
RequestHandle &operator=(RequestHandle &&other) = delete
RequestHandle(RequestHandle const&) = delete
RequestHandle &operator=(RequestHandle const&) = delete
~RequestHandle() noexcept
bool valid() const noexcept
struct Impl

Public Types

enum class Phase : uint8_t

Values:

enumerator kAdmitted
enumerator kExecuting
enumerator kFinishing

Public Functions

inline Impl(BoundedSwaKVPageManager &manager_, cudaStream_t stream_)

Public Members

BoundedSwaKVPageManager *manager = {}
cudaStream_t stream = {}
std::vector<int32_t> inputLengths
std::vector<SwaKVCacheState> states
std::vector<int32_t> pendingCompactionMapping
int32_t pendingCompactionBatchSize = {-1}
Tensor const *pendingDeviceBatchMapping = {}
Phase phase = {Phase::kAdmitted}
bool deviceWorkPending = {}
struct ExternalPageLifecycle

Hooks used only when a context-cache lease owns the request’s page references.

Public Members

std::function<bool(int32_t, PageId)> isExternallyOwned
std::function<PageId(int32_t, PageId)> replace
struct BeginRequestResult

Public Members

SwaKVCacheStatus status = {SwaKVCacheStatus::kRequestFailed}
std::optional<RequestHandle> request