Bounded Swa KV Page Manager

class BoundedSwaKVPageManager

Runtime-level allocator and execution-lifecycle owner for bounded SWA pages.

KVCacheManager owns the device buffers. This class owns only page IDs, sparse-table mappings, and request-local transition state. Pages staged for retirement remain leased until the existing CUDA completion point.

Public Types

using StreamSynchronizer = std::function<cudaError_t(cudaStream_t)>

Public Functions

BoundedSwaKVPageManager(

HybridCacheManager &baseCache,

KVPageTable &basePageTable,

KVPageTable &swaPageTable,

cudaStream_t stream,

StreamSynchronizer synchronizer = {}

)

~BoundedSwaKVPageManager() noexcept

BoundedSwaKVPageManager(BoundedSwaKVPageManager const&) = delete

BoundedSwaKVPageManager &operator=(

BoundedSwaKVPageManager const&

) = delete

BeginRequestResult beginRequest(

std::vector<ResidentRef> const &residents,

cudaStream_t stream

)

SwaKVCacheStatus preparePrefill(

RequestHandle &request,

std::vector<int32_t> const &inputLengths

)

SwaKVCacheStatus completePrefill(RequestHandle &request)

SwaKVCacheStatus prepareDecodeStep(RequestHandle &request)

SwaKVCacheStatus completeDecodeStep(RequestHandle &request)

SwaKVCacheStatus beginBatchCompaction(

RequestHandle &request,

std::vector<int32_t> const &oldToNew,

int32_t newBatchSize,

Tensor &deviceBatchMapping

)

SwaKVCacheStatus compactBatch(RequestHandle &request)

SwaKVCacheStatus finish(RequestHandle &request)

SwaKVCacheStatus shutdown() noexcept

int32_t freePageCount() const noexcept

int32_t privatePagesPerSequence() const noexcept

SwaKVCacheState const &state(

RequestHandle const &request,

int32_t slot

) const

struct BeginRequestResult

Public Members

SwaKVCacheStatus status = {SwaKVCacheStatus::kRequestFailed}

std::optional<RequestHandle> request

class RequestHandle

Public Functions

RequestHandle(RequestHandle &&other) noexcept

RequestHandle &operator=(RequestHandle &&other) = delete

RequestHandle(RequestHandle const&) = delete

RequestHandle &operator=(RequestHandle const&) = delete

~RequestHandle() noexcept

bool valid() const noexcept

struct Impl

Public Types

enum class Phase : uint8_t

Values:

enumerator kAdmitted

enumerator kExecuting

enumerator kFinishing

Public Functions

inline Impl(BoundedSwaKVPageManager &manager_, cudaStream_t stream_)

Public Members

BoundedSwaKVPageManager *manager = {}

cudaStream_t stream = {}

std::vector<int32_t> inputLengths

std::vector<SwaKVCacheState> states

std::vector<int32_t> pendingCompactionMapping

int32_t pendingCompactionBatchSize = {-1}

Tensor const *pendingDeviceBatchMapping = {}

Phase phase = {Phase::kAdmitted}

bool deviceWorkPending = {}

class RequestHandle

Public Functions

RequestHandle(RequestHandle &&other) noexcept

RequestHandle &operator=(RequestHandle &&other) = delete

RequestHandle(RequestHandle const&) = delete

RequestHandle &operator=(RequestHandle const&) = delete

~RequestHandle() noexcept

bool valid() const noexcept

struct Impl

Public Types

enum class Phase : uint8_t

Values:

enumerator kAdmitted

enumerator kExecuting

enumerator kFinishing

Public Functions

inline Impl(BoundedSwaKVPageManager &manager_, cudaStream_t stream_)

Public Members

BoundedSwaKVPageManager *manager = {}

cudaStream_t stream = {}

std::vector<int32_t> inputLengths

std::vector<SwaKVCacheState> states

std::vector<int32_t> pendingCompactionMapping

int32_t pendingCompactionBatchSize = {-1}

Tensor const *pendingDeviceBatchMapping = {}

Phase phase = {Phase::kAdmitted}

bool deviceWorkPending = {}

struct SwaPageBinding

One absolute logical-to-physical binding in a bounded SWA window.

Public Members

int32_t logicalPage = {}

PageId page = {}

struct SwaKVCacheState

Request-local execution state for one bounded SWA sequence.

Logical positions and the completed endpoint remain absolute. The compact physical bindings are only an execution view and never identify a reusable prefix. Context reuse may seed or consume settled state in a later integration, but it does not own window rotation.

Public Members

ResidentRef resident

int32_t pageSizeTokens = {}

int32_t windowSizeTokens = {}

int32_t exactResidentTokenCount = {}

int32_t retainedLogicalPageBegin = {}

int32_t retainedLogicalPageEnd = {}

std::deque<SwaPageBinding> pageBindings

std::vector<PageId> reservedPages

std::vector<SwaPageBinding> newPageBindings

std::vector<SwaPageBinding> pendingRetireBindings

std::optional<int32_t> pendingEndpoint

struct BeginRequestResult

Public Members

SwaKVCacheStatus status = {SwaKVCacheStatus::kRequestFailed}

std::optional<RequestHandle> request