LLM Rank Runtime#

class LLMRankRuntime#

Public Types

using TokenBroadcastFn = std::function<bool(void *buffer, int32_t count, cudaStream_t stream)>#

Callback type for broadcasting GPU int32 buffers across parallel ranks. Rank 0 sends; peer ranks receive.

using VisionGatherFn = rt::VisionGatherFn#

Gather two FP16 visual shards already placed at their final offsets in the output buffer.

Public Functions

LLMRankRuntime(
DeploymentOptions const &options,
cudaStream_t stream,
ParallelMapping const &mapping,
tokenizer::Tokenizer &tokenizer,
chat_template::ChatTemplate const &chatTemplate,
std::shared_ptr<PleIoBudget> pleIoBudget = nullptr
)#

Construct one rank-local runtime from a fully-resolved parallel mapping. Preferred entry point — carries tensor/context/expert coordinates so future CP/EP support needs no further constructor changes.

LLMRankRuntime(
ModelArtifacts &&artifacts,
DeploymentOptions const &options,
cudaStream_t stream,
ParallelMapping const &mapping,
tokenizer::Tokenizer &tokenizer,
chat_template::ChatTemplate const &chatTemplate,
std::shared_ptr<PleIoBudget> pleIoBudget = nullptr
)#
~LLMRankRuntime()#

Destructor.

bool captureDecodingCUDAGraph(cudaStream_t stream)#

Capture CUDA graphs for decoding stages to optimize performance.

When draft model is present, captures graphs for draft proposal, draft accept token, base verification, and base vanilla decoding. Without draft model, captures only vanilla decoding graphs.

Note

If capture fails for any stage, the inference can proceed without CUDA graph capture, but at cost of performance degradation.

Parameters:

stream – CUDA stream

Throws:

std::runtime_error – if a tensor reshape operation fails

Returns:

True if all stage captures succeed, false otherwise

bool handleRequest(
LLMGenerationRequest const &request,
LLMGenerationResponse &response,
cudaStream_t stream,
bool outputThinkerEmbeddings = false,
TokenBroadcastFn tokenBroadcast = nullptr,
VisionGatherFn visionGather = nullptr,
int32_t parallelRank = -1
)#
inline void setActionKvAllGather(
Alpamayo1ActionRunner::KvAllGatherFn kvAllGather
)#

Install the rank-local TP collective used by the optional Alpamayo action bridge.

std::unique_ptr<SteppedGeneration> beginGeneration(
LLMGenerationRequest const &request,
LLMGenerationResponse &response,
cudaStream_t stream,
bool outputThinkerEmbeddings,
TokenBroadcastFn tokenBroadcast,
VisionGatherFn visionGather,
int32_t parallelRank
)#

Everything handleRequest does before the generation loop: validation, context population, the context-cache admit, prefill-execution setup, streaming setup, and the request’s prefill itself. Returns null on any refusal (the same conditions that returned false), and throws where handleRequest threw. request must outlive the returned object.

bool finishGeneration(
SteppedGeneration &generation,
LLMGenerationRequest const &request,
LLMGenerationResponse &response,
cudaStream_t stream
)#

Everything handleRequest does after the loop: the drained-batch check, the context-cache finish, metrics, and response assembly from whatever the loop’s harvests left in completedBatches.

inline std::string unavailableReason() const#

Empty while reusable; the first failed CUDA drain otherwise. Safe to read while a request runs.

std::unique_ptr<RuntimeSession> openSession(
cudaStream_t stream,
SessionOptions options = {}
)#

Open the runtime session: one long-lived generation that sequences enter and leave one at a time (see RuntimeSession).

Takes the one-request latch for the session’s life, so handleRequest() is refused until the session is destroyed. Returns null when the runtime is already serving a request, or on the deployment refusals a request would also have met (an unavailable LoRA reset, a context-cache request that could not be opened, visual-token pruning under managed reuse). Deployments whose sequences cannot be seated one at a time (a diffusion backbone, bounded SWA storage, a context cache without live admission) cannot host a session: check supportsSeatedAdmission() first. options choose whether the session may run compact steps (see SessionOptions).

void setActionNoiseSeed(int32_t seed) noexcept#

Set the random seed used when initializing the action diffusion noise trajectory.

Parameters:

seed – Random seed value; has no effect if no action runner is loaded

void setVisualPrunerConfig(VisualPrunerConfig const &config)#

Enable visual-token pruning for supported VLM prefill execution.

inline metrics::LLMPrefillMetrics const &getPrefillMetrics(
) const noexcept#

Get LLM prefill stage metrics.

inline metrics::SpecDecodeGenerationMetrics const &getSpecDecodeGenerationMetrics(
) const noexcept#

Get speculative decoding generation stage metrics (only meaningful when draft model is present)

inline char const *getSpeculativeDecodingStrategyName(
) const noexcept#
inline char const *getDefaultDecodingStrategyName() const noexcept#
inline metrics::LLMGenerationMetrics const &getGenerationMetrics(
) const noexcept#

Get vanilla generation stage metrics (only meaningful when no draft model / vanilla path)

std::optional<ContextCacheMetrics> getContextCacheMetrics(
) const noexcept#

Get context-cache metrics, or nullopt when the runtime cache is disabled.

inline metrics::MultimodalMetrics getMultimodalMetrics(
) const noexcept#

Get multimodal metrics (returns empty metrics if no multimodal runner)

inline rt::Tensor const &getEmbeddingTable() const#

Get the embedding table (for Talker streaming pipeline)

inline rt::Tensor const *getBaseModelHiddenStates(
int32_t layerIdx
) const noexcept#

Get a base model hidden-states buffer for the requested layer index.

Buffers are owned by the runtime and reused across requests. Layer 0 corresponds to the post-multimodal input embeddings (backed up before the decode loop reshapes them); other layer indices correspond to engine-output hidden states (e.g. acceptHiddenLayer for the Qwen3-Omni Talker, or future MTP layers).

Lifetime contract:

  • Buffers are sized to {maxRuntimeBatchSize, maxSupportedInputLength, hiddenSize}.

  • Contents are cleared (overwritten) at the start of each handleRequest() call and remain valid until the next handleRequest() begins. The buffer is reshaped to {activeBatchSize, prefillLength, hiddenSize} for the most recent request — use getBaseModelPrefillLength() to query the valid prefill length.

  • The caller is responsible for consuming the data within that window.

Parameters:

layerIdx – Layer index. 0 = input embeddings (post-multimodal); other indices are model-specific (e.g. acceptHiddenLayer for Qwen3-Omni Talker).

Returns:

Pointer to the buffer, or nullptr if no buffer is registered for that layer.

inline int32_t getBaseModelPrefillLength() const noexcept#

Number of valid prefill tokens in the hidden-states buffers from the most recent handleRequest() call. Returns 0 if no hidden-states output was requested.

inline std::vector<std::vector<int32_t>> const &getBaseModelInputTokenIds(
) const noexcept#

Per-batch input token IDs from the most recent handleRequest() call. Cleared at the start of each handleRequest(); valid until the next one begins.

inline bool hasDraftModel() const noexcept#

Check if draft model is loaded and spec-decode is available.

bool supportsSeatedAdmission() const noexcept#

Whether this deployment can take boundary admissions via prefillSlotInPlace.

False for the static refusals prefillSlotInPlace would otherwise throw on mid-request: diffusion backbones, bounded-SWA deployments, and a context cache without live admission. Probed once at RequestEngine construction so an unsupported deployment falls back to the blocking path instead of failing admissions.

size_t activeCacheManagerCount(
DecodingStrategy const &strategy
) const noexcept#

How many of the shared cache managers strategy drives: the base one, plus the draft one when the strategy is speculative. The batch of each of them follows the context’s.

inline void selectProfilingTimer() const#

Select this runtime’s statistics for legacy formatter entry points on the calling thread.

int32_t maxBatchSize() const noexcept#

The batch dimension the engine was built with: the physical bound on how many sequences a batch can seat, and therefore on any scheduler’s maxBatchSize.

class GenerationSession#

Handle generation request.

One request’s generation, advanced a step at a time.

Exists because the decode loop carries state between steps: a thinking-done flag per slot, several tokenizer ids, and a stop predicate that closes over them. While those were locals in one long function, that state and the loop were forced to share a lifetime by construction. As an object they still share one, but it is now the object’s, and a caller can hold it across steps instead of being obliged to run the loop to the end in a single call.

The prefill preceding the first step also produces a token on most backbones, so a session is primed once before it is stepped – see primeFromPrefill().

Note

Calls on the same runtime must be externally serialized. An accidental overlap with another handleRequest() is rejected before runtime or response state is mutated; this is not a general thread-safety guarantee.

Param request:

Generation request with prompts and parameters

Param response:

Output response with generated tokens and text

Param stream:

CUDA stream

Throws std::runtime_error:

if an LLM or CUDA operation fails

Return:

True on success, false on failure

Public Functions

GenerationSession(
LLMRankRuntime &runtime,
DecodingInferenceContext &context,
DecodingStrategy &strategy,
ManagedKVCacheRequest *managedRequest,
DecodingKvHeadroom const &kvHeadroom,
DecodingKvHeadroom const &jumpForwardKvHeadroom,
cudaStream_t stream,
bool synchronizeCancellation,
std::vector<bool> actionSlots
)#
Parameters:
  • synchronizeCancellation – Run the cross-rank cancellation consensus at every step. The blocking path needs it only when the request streams (a consumer can cancel the channel); a scheduled session needs it always, since any resident’s channel may be cancelled and there is no single request whose channels would decide.

  • actionSlots – Per original index, whether that sequence carries a trajectory request (Alpamayo); empty when none does. Indexed by batchIndexMapping.

  • jumpForwardKvHeadroom – The headroom of a slot that runs with jump-forward; equal to kvHeadroom when no slot does or the strategy does not widen its rounds for a jump.

~GenerationSession()#

Clears the stop predicate the context holds, which closes over members of this object.

GenerationSession(GenerationSession const&) = delete#
GenerationSession &operator=(GenerationSession const&) = delete#
bool primeFromPrefill()#

Consume the token prefill produced, before any decode step runs.

Prefill emits a token on every backbone except the diffusion one, and that token has to travel the same cancel/decode/finalize/emit path a decode step would give it. Skipping this drops the first token of every request.

bool finished() const#

True once every slot has reached a terminal state, or none are left.

bool advance()#

Advance every live slot by one decode step.

std::unordered_map<int32_t, BatchResult> takeCompletedAtOrAbove(
int32_t firstIndex
)#

Move out the results of finished sequences whose original index is >= firstIndex.

Results are filed under the original index a sequence was seated with; the session harvests them all (firstIndex 0) at every step it runs.

LLMGenerationResponse materializeResult(
BatchResult const &result,
std::vector<std::string> const &stopStrings
) const#

Turn one finished sequence’s result into the caller-facing response, decoding included, exactly as the blocking path’s own assembly would have.

struct InProgressGuard#

Releases the runtime’s one-request-at-a-time latch, whatever path retires the request or closes the session that held it. Acquisition is the owner’s compare-exchange; this only guarantees the release.

Public Functions

inline explicit InProgressGuard(
std::atomic<bool> &active
) noexcept#
inline ~InProgressGuard() noexcept#
InProgressGuard(InProgressGuard const&) = delete#
InProgressGuard &operator=(InProgressGuard const&) = delete#

Public Members

std::atomic<bool> &mActive#
struct SteppedGeneration#

Everything one request owns between beginGeneration and finishGeneration: the same state handleRequest used to keep on its stack, packaged so it can be held across the generation loop. The request must outlive this object; member order is load-bearing (the stream finalizer references the context and must be destroyed first).

Public Functions

inline explicit SteppedGeneration(
std::atomic<bool> &activeFlag,
Tensor &hostTokenStorage,
DeviceFailureState &failure,
cudaStream_t stream
) noexcept#
SteppedGeneration(SteppedGeneration const&) = delete#
SteppedGeneration &operator=(SteppedGeneration const&) = delete#
~SteppedGeneration() noexcept#
inline ManagedKVCacheRequest *managedRequest() noexcept#

Public Members

InProgressGuard guard#
DecodingInferenceContext context#
DecodingStrategy *strategy = {nullptr}#
std::optional<ManagedKVCacheRequest> managedKVCacheRequest#
DecodingKvHeadroom kvHeadroom = {}#
DecodingKvHeadroom jumpForwardKvHeadroom = {}#

Headroom of the slots that run with jump-forward; kvHeadroom when none does.

std::optional<StreamChannelFinalizer> streamFinalizer#
bool enableSpecDecode = {false}#
bool hasActionRequest = {false}#
ActionKvBatchCollector *actionKvCollector = {nullptr}#
bool completed = {false}#
class GenerationSession

Handle generation request.

One request’s generation, advanced a step at a time.

Exists because the decode loop carries state between steps: a thinking-done flag per slot, several tokenizer ids, and a stop predicate that closes over them. While those were locals in one long function, that state and the loop were forced to share a lifetime by construction. As an object they still share one, but it is now the object’s, and a caller can hold it across steps instead of being obliged to run the loop to the end in a single call.

The prefill preceding the first step also produces a token on most backbones, so a session is primed once before it is stepped – see primeFromPrefill().

Note

Calls on the same runtime must be externally serialized. An accidental overlap with another handleRequest() is rejected before runtime or response state is mutated; this is not a general thread-safety guarantee.

Param request:

Generation request with prompts and parameters

Param response:

Output response with generated tokens and text

Param stream:

CUDA stream

Throws std::runtime_error:

if an LLM or CUDA operation fails

Return:

True on success, false on failure

Public Functions

GenerationSession(
LLMRankRuntime &runtime,
DecodingInferenceContext &context,
DecodingStrategy &strategy,
ManagedKVCacheRequest *managedRequest,
DecodingKvHeadroom const &kvHeadroom,
DecodingKvHeadroom const &jumpForwardKvHeadroom,
cudaStream_t stream,
bool synchronizeCancellation,
std::vector<bool> actionSlots
)
Parameters:
  • synchronizeCancellation – Run the cross-rank cancellation consensus at every step. The blocking path needs it only when the request streams (a consumer can cancel the channel); a scheduled session needs it always, since any resident’s channel may be cancelled and there is no single request whose channels would decide.

  • actionSlots – Per original index, whether that sequence carries a trajectory request (Alpamayo); empty when none does. Indexed by batchIndexMapping.

  • jumpForwardKvHeadroom – The headroom of a slot that runs with jump-forward; equal to kvHeadroom when no slot does or the strategy does not widen its rounds for a jump.

~GenerationSession()

Clears the stop predicate the context holds, which closes over members of this object.

GenerationSession(GenerationSession const&) = delete
GenerationSession &operator=(GenerationSession const&) = delete
bool primeFromPrefill()

Consume the token prefill produced, before any decode step runs.

Prefill emits a token on every backbone except the diffusion one, and that token has to travel the same cancel/decode/finalize/emit path a decode step would give it. Skipping this drops the first token of every request.

bool finished() const

True once every slot has reached a terminal state, or none are left.

bool advance()

Advance every live slot by one decode step.

std::unordered_map<int32_t, BatchResult> takeCompletedAtOrAbove(
int32_t firstIndex
)

Move out the results of finished sequences whose original index is >= firstIndex.

Results are filed under the original index a sequence was seated with; the session harvests them all (firstIndex 0) at every step it runs.

LLMGenerationResponse materializeResult(
BatchResult const &result,
std::vector<std::string> const &stopStrings
) const

Turn one finished sequence’s result into the caller-facing response, decoding included, exactly as the blocking path’s own assembly would have.

struct InProgressGuard

Releases the runtime’s one-request-at-a-time latch, whatever path retires the request or closes the session that held it. Acquisition is the owner’s compare-exchange; this only guarantees the release.

Public Functions

inline explicit InProgressGuard(
std::atomic<bool> &active
) noexcept
inline ~InProgressGuard() noexcept
InProgressGuard(InProgressGuard const&) = delete
InProgressGuard &operator=(InProgressGuard const&) = delete

Public Members

std::atomic<bool> &mActive
struct SteppedGeneration

Everything one request owns between beginGeneration and finishGeneration: the same state handleRequest used to keep on its stack, packaged so it can be held across the generation loop. The request must outlive this object; member order is load-bearing (the stream finalizer references the context and must be destroyed first).

Public Functions

inline explicit SteppedGeneration(
std::atomic<bool> &activeFlag,
Tensor &hostTokenStorage,
DeviceFailureState &failure,
cudaStream_t stream
) noexcept
SteppedGeneration(SteppedGeneration const&) = delete
SteppedGeneration &operator=(SteppedGeneration const&) = delete
~SteppedGeneration() noexcept
inline ManagedKVCacheRequest *managedRequest() noexcept

Public Members

InProgressGuard guard
DecodingInferenceContext context
DecodingStrategy *strategy = {nullptr}
std::optional<ManagedKVCacheRequest> managedKVCacheRequest
DecodingKvHeadroom kvHeadroom = {}
DecodingKvHeadroom jumpForwardKvHeadroom = {}

Headroom of the slots that run with jump-forward; kvHeadroom when none does.

std::optional<StreamChannelFinalizer> streamFinalizer
bool enableSpecDecode = {false}
bool hasActionRequest = {false}
ActionKvBatchCollector *actionKvCollector = {nullptr}
bool completed = {false}