LLM Inference Runtime#

class LLMInferenceRuntime#

Public LLM inference runtime for single-rank and multi-rank execution.

LLMInferenceRuntime is the stable user-facing API. It delegates execution to a RuntimeCoordinator, which resolves the parallel plan and owns one or more internal LLMRankRuntime instances. Single-device execution is represented as a size-1 parallel plan rather than a separate runtime implementation.

Public Functions

LLMInferenceRuntime(DeploymentOptions options, cudaStream_t stream)#

Construct a single-device runtime using a caller-owned stream.

explicit LLMInferenceRuntime(
DeploymentOptions options,
RuntimeResources resources = {}
)#

Construct a runtime with explicit rank placement and execution-resource ownership.

LLMInferenceRuntime(
ModelArtifacts &&artifacts,
DeploymentOptions options,
cudaStream_t stream
)#

Assemble using artifact-owned model/source configuration and caller-owned runtime policies. Nonempty load options must agree with the artifacts; omitted drafting inherits their resolved mode.

LLMInferenceRuntime(
std::string const &engineDir,
std::string const &multimodalEngineDir,
std::unordered_map<std::string, std::string> const &loraWeightsMap,
SpecDecodeDraftingConfig const &draftingConfig,
cudaStream_t stream,
ContextCacheConfig const &contextCacheConfig = {},
std::string const &checkpointDir = "",
std::string const &draftCheckpointDir = "",
std::optional<std::filesystem::path> const &runtimeCachePath = std::nullopt,
PleRuntimeConfig const &pleConfig = {}
)#
LLMInferenceRuntime(
std::string const &engineDir,
std::string const &multimodalEngineDir,
std::unordered_map<std::string, std::string> const &loraWeightsMap,
cudaStream_t stream,
ContextCacheConfig const &contextCacheConfig = {},
std::string const &checkpointDir = "",
std::optional<std::filesystem::path> const &runtimeCachePath = std::nullopt,
PleRuntimeConfig const &pleConfig = {}
)#
LLMInferenceRuntime(
ModelArtifacts &&artifacts,
std::string const &engineDir,
std::string const &multimodalEngineDir,
std::unordered_map<std::string, std::string> const &loraWeightsMap,
std::optional<SpecDecodeDraftingConfig> const &draftingConfig,
cudaStream_t stream,
ContextCacheConfig const &contextCacheConfig = {}
)#
~LLMInferenceRuntime() noexcept#
bool captureDecodingCUDAGraph(cudaStream_t stream)#
bool handleRequest(
LLMGenerationRequest const &request,
LLMGenerationResponse &response,
cudaStream_t stream,
bool outputThinkerEmbeddings = false
)#
std::string unavailableReason() const#

Empty while reusable; a permanent failure reason until the process restarts otherwise.

std::vector<int32_t> countPromptTokens(
LLMGenerationRequest const &request
) const#

Return the input size for an explicit text token-count request.

void setActionNoiseSeed(int32_t seed)#
void setVisualPrunerConfig(VisualPrunerConfig const &config)#

Enable visual-token pruning for supported single-device VLM execution.

metrics::LLMPrefillMetrics const &getPrefillMetrics() const#
metrics::SpecDecodeGenerationMetrics const &getSpecDecodeGenerationMetrics(
) const#
char const *getSpeculativeDecodingStrategyName() const#
char const *getDefaultDecodingStrategyName() const#
metrics::LLMGenerationMetrics const &getGenerationMetrics() const#
std::optional<ContextCacheMetrics> getContextCacheMetrics() const#
metrics::MultimodalMetrics getMultimodalMetrics() const#
rt::Tensor const &getEmbeddingTable() const#
rt::Tensor const *getBaseModelHiddenStates(int32_t layerIdx) const#
int32_t getBaseModelPrefillLength() const#
std::vector<std::vector<int32_t>> const &getBaseModelInputTokenIds(
) const#
bool hasDraftModel() const#
bool supportsSeatedAdmission() const#

Whether the deployment can take boundary admissions (see LLMRankRuntime).

int32_t maxBatchSize() const#

The engine’s built batch dimension (see LLMRankRuntime::maxBatchSize).

bool supportsSteppedExecution() const noexcept#

True when this runtime can host a runtime session (inline single-rank only in this release).

std::unique_ptr<RuntimeSession> openSession(
cudaStream_t stream,
SessionOptions options = {}
)#

Open the runtime session a scheduler drives for the engine’s life; see RuntimeSession and RuntimeCoordinator::openSession. Null when the runtime refuses: a request is in flight, the deployment cannot seat sequences into a running batch (supportsSeatedAdmission), or the session’s own setup fails. options choose whether the session may run compact steps (see SessionOptions).

int32_t worldSize() const noexcept#

Ranks this runtime spans. 1 when it runs on a single device.

Exposed because a caller’s own structure can depend on it: a scheduler that varies which requests take part in a step has to keep that decision identical on every rank, or the collectives inside a forward pass disagree on their shapes.

bool ownsGlobalRank(int32_t globalRank) const noexcept#

True when this runtime instance owns the requested global rank.