Context Cache Coordinator#

class ContextCacheCoordinator#

Owns the complete host/device lifecycle around the CUDA-free ContextCacheManager.

The coordinator has no worker and is deliberately single-request-at-a-time under the runtime’s serialized request contract. Normal publication occurs only at host-visible completion points already present in the runtime. An abnormal exit drains the bound stream before releasing active page references; a failed drain quarantines the entire request and poisons the coordinator until runtime-owned shutdown can establish quiescence.

Public Types

using StreamSynchronizer = std::function<cudaError_t(cudaStream_t)>#

Public Functions

ContextCacheCoordinator( ContextCacheConfig const &config, DeploymentConfig const &deployment, ContextCacheDeploymentKind deploymentKind, ContextCachePhysicalResources resources, cudaStream_t stream, StreamSynchronizer synchronizer = {} )#

~ContextCacheCoordinator() noexcept#

ContextCacheCoordinator(ContextCacheCoordinator const&) = delete#

ContextCacheCoordinator &operator=( ContextCacheCoordinator const& ) = delete#

BeginRequestResult beginRequest( ContextCacheBatchAdmission const &admission, cudaStream_t stream )#

ContextCacheCoordinatorStatus preparePrefill(RequestHandle &request)#: Bind every admitted row and reset logical cache lengths to the selected reuse boundaries.

ContextCacheCoordinatorStatus enqueuePrefillCaptures( RequestHandle &request )#: Reserve and enqueue hybrid prefill-end snapshots before the runtime’s existing prefill synchronization.

ContextCacheCoordinatorStatus finalizePrefillPublication( RequestHandle &request, std::vector<ContextCacheSequenceAdvance> const &advances, std::vector<int32_t> const *commonStateLengths = nullptr )#: Apply the post-prefill sequence advance and publish every ready full-block endpoint. For EAGLE, commonStateLengths caps publication at the prefix materialized by both base and draft state.

ContextCacheCoordinatorStatus prepareDecodeStep( RequestHandle &request )#: Grow and upload every row needed for the next decode working set before model execution.

ContextCacheCoordinatorStatus completeDecodeStep( RequestHandle &request, std::vector<ContextCacheSequenceAdvance> const &advances, std::vector<int32_t> const &publishableCompletedSlots, std::vector<int32_t> const *commonStateLengths = nullptr )#: Apply the post-decode sequence advance after the decoder’s existing synchronization. For EAGLE, commonStateLengths excludes any unmaterialized accepted suffix and speculative lookahead.

ContextCacheCoordinatorStatus beginBatchCompaction( RequestHandle &request, std::vector<int32_t> const &oldToNew, int32_t newBatchSize, Tensor &deviceBatchMapping )#: Validate and upload the one authoritative old-to-new mapping before any old-slot compaction work.

ContextCacheCoordinatorStatus compactBatch(RequestHandle &request)#: Compact slot-addressed state/page-table rows, retire leases, and consume the existing eviction sync.

ContextCacheCoordinatorStatus finish(RequestHandle &request)#: Consume a normally completed request. This is idempotent for an already-empty handle.

ContextCacheCoordinatorStatus shutdown() noexcept#: Drain quarantined ownership before physical cache resources are destroyed.

ContextCacheMetrics metrics() const noexcept#

ContextCacheManager const &manager() const noexcept#

struct AcquireSequenceResult#

Public Members

std::optional<CacheRequestLease> lease#

AcquireStatus status = {AcquireStatus::kInsufficientCapacity}#

ReusePlan plan#

bool forcedCold = {}#

struct AdmissionResult#

Public Members

RequestHandle request#

std::vector<int32_t> prefillStarts#: Per-sequence logical token offset at which runtime prefill begins.

struct BeginRequestResult#

Public Members

ContextCacheCoordinatorStatus status = {ContextCacheCoordinatorStatus::kRequestFailed}#

std::optional<AdmissionResult> admission#

class RequestHandle#

Public Functions

RequestHandle(RequestHandle &&other) noexcept#

RequestHandle &operator=(RequestHandle &&other) = delete#

RequestHandle(RequestHandle const&) = delete#

RequestHandle &operator=(RequestHandle const&) = delete#

~RequestHandle() noexcept#

bool valid() const noexcept#

struct Impl#

Public Types

enum class Phase : uint8_t#

Values:

enumerator kAdmitted#

enumerator kExecuting#

enumerator kFinishing#

Public Functions

inline explicit Impl( ContextCacheCoordinator &coordinator, cudaStream_t requestStream ) noexcept#

Public Members

RequestSlotToken requestSlot#: Declared before sequences so its destructor clears the admission flag only after every lease is released.

ContextCacheCoordinator *owner = {}#

cudaStream_t stream = {}#

ContextCacheExecutionMode executionMode = {ContextCacheExecutionMode::kVanilla}#

Phase phase = {Phase::kAdmitted}#

bool deviceWorkPending = {}#

bool specAwaitingFirstCompletion = {}#

std::vector<int32_t> pendingCompactionMapping#

int32_t pendingCompactionBatchSize = {-1}#

Tensor const *pendingDeviceBatchMapping = {}#

std::vector<SequenceState> sequences#

struct RequestSlotToken#

Public Functions

inline explicit RequestSlotToken( ContextCacheCoordinator &coordinator ) noexcept#

inline RequestSlotToken(RequestSlotToken &&other) noexcept#

RequestSlotToken &operator=(RequestSlotToken&&) = delete#

RequestSlotToken(RequestSlotToken const&) = delete#

RequestSlotToken &operator=(RequestSlotToken const&) = delete#

inline ~RequestSlotToken() noexcept#

Public Members

ContextCacheCoordinator *owner = {}#

struct SequenceState#

Public Members

CacheRequestLease lease#

std::vector<int32_t> tokenIds#

BlockKeyExtras keyExtras#

std::vector<Hash128> perPositionMediaHash#

int32_t reuseTokenLength = {}#

ContextCacheLookupPolicy lookupPolicy = {ContextCacheLookupPolicy::kUseCache}#

ContextCacheCommitPolicy commitPolicy = {ContextCacheCommitPolicy::kIncludingGeneratedTokens}#

int32_t committedStateLength = {}#

int32_t publishedFullBlockCount = {}#

int32_t publishedExactLength = {}#

int32_t commonStateLength = {}#

std::optional<int32_t> frozenSpecPrefillLength#

std::optional<StagedHybridPublication> stagedHybridPublication#

struct StagedHybridPublication#

Public Members

HybridCheckpointKey checkpoint#

HybridSnapshotReservation snapshots#

PublicationPoint point = {}#

class RequestHandle

Public Functions

RequestHandle(RequestHandle &&other) noexcept

RequestHandle &operator=(RequestHandle &&other) = delete

RequestHandle(RequestHandle const&) = delete

RequestHandle &operator=(RequestHandle const&) = delete

~RequestHandle() noexcept

bool valid() const noexcept

struct Impl

Public Types

enum class Phase : uint8_t

Values:

enumerator kAdmitted

enumerator kExecuting

enumerator kFinishing

Public Functions

inline explicit Impl( ContextCacheCoordinator &coordinator, cudaStream_t requestStream ) noexcept

Public Members

RequestSlotToken requestSlot: Declared before sequences so its destructor clears the admission flag only after every lease is released.

ContextCacheCoordinator *owner = {}

cudaStream_t stream = {}

ContextCacheExecutionMode executionMode = {ContextCacheExecutionMode::kVanilla}

Phase phase = {Phase::kAdmitted}

bool deviceWorkPending = {}

bool specAwaitingFirstCompletion = {}

std::vector<int32_t> pendingCompactionMapping

int32_t pendingCompactionBatchSize = {-1}

Tensor const *pendingDeviceBatchMapping = {}

std::vector<SequenceState> sequences

struct RequestSlotToken

Public Functions

inline explicit RequestSlotToken( ContextCacheCoordinator &coordinator ) noexcept

inline RequestSlotToken(RequestSlotToken &&other) noexcept

RequestSlotToken &operator=(RequestSlotToken&&) = delete

RequestSlotToken(RequestSlotToken const&) = delete

RequestSlotToken &operator=(RequestSlotToken const&) = delete

inline ~RequestSlotToken() noexcept

Public Members

ContextCacheCoordinator *owner = {}

struct SequenceState

Public Members

CacheRequestLease lease

std::vector<int32_t> tokenIds

BlockKeyExtras keyExtras

std::vector<Hash128> perPositionMediaHash

int32_t reuseTokenLength = {}

ContextCacheLookupPolicy lookupPolicy = {ContextCacheLookupPolicy::kUseCache}

ContextCacheCommitPolicy commitPolicy = {ContextCacheCommitPolicy::kIncludingGeneratedTokens}

int32_t committedStateLength = {}

int32_t publishedFullBlockCount = {}

int32_t publishedExactLength = {}

int32_t commonStateLength = {}

std::optional<int32_t> frozenSpecPrefillLength

std::optional<StagedHybridPublication> stagedHybridPublication

struct StagedHybridPublication

Public Members

HybridCheckpointKey checkpoint

HybridSnapshotReservation snapshots

PublicationPoint point = {}

struct ContextCachePhysicalResources#

Validated physical resources borrowed from SharedResources for the coordinator lifetime.

Public Members

HybridCacheManager &baseCache#

KVPageTable &basePageTable#

HybridCacheManager *draftCache = {}#

KVPageTable *draftPageTable = {}#

struct ContextCacheSequenceAdmission#

One complete logical input admitted to the cache before its executable suffix is derived.

Public Members

std::vector<int32_t> tokenIds#

BlockKeyExtras keyExtras#: Request-wide non-token identity; LoRA/isolation identity is constant for the sequence.

std::vector<Hash128> perPositionMediaHash#: Per-position media content hash. Empty means text-only. When non-empty, must have tokenIds.size() entries. A non-zero Hash128 at position i causes the block hash to consume that 128-bit digest instead of the token ID.

struct ContextCacheBatchAdmission#

One serialized runtime request. Bypass still uses managed private pages but neither looks up nor publishes state.

Public Members

std::vector<ContextCacheSequenceAdmission> sequences#

ContextCacheExecutionMode executionMode = {ContextCacheExecutionMode::kVanilla}#

ContextCacheLookupPolicy lookupPolicy = {ContextCacheLookupPolicy::kUseCache}#

ContextCacheCommitPolicy commitPolicy = {ContextCacheCommitPolicy::kIncludingGeneratedTokens}#

struct ContextCacheSequenceAdvance#

Host-visible sequence advance observed after an existing stream synchronization.

Public Members

int32_t const *acceptedTokenIds = {}#

int32_t acceptedTokenCount = {}#

int32_t committedStateLength = {}#: Greatest logical token boundary whose model state is materialized in the bound cache.

struct AdmissionResult

Public Members

RequestHandle request

std::vector<int32_t> prefillStarts: Per-sequence logical token offset at which runtime prefill begins.

struct BeginRequestResult

Public Members

ContextCacheCoordinatorStatus status = {ContextCacheCoordinatorStatus::kRequestFailed}

std::optional<AdmissionResult> admission