LLM Inference Runtime

class LLMInferenceRuntime

Public LLM inference runtime for single-rank and multi-rank execution.

LLMInferenceRuntime is the stable user-facing API. It delegates execution to a RuntimeCoordinator, which resolves the parallel plan and owns one or more internal LLMRankRuntime instances. Single-device execution is represented as a size-1 parallel plan rather than a separate runtime implementation.

Public Functions

LLMInferenceRuntime(

std::string const &engineDir,

std::string const &multimodalEngineDir,

std::unordered_map<std::string, std::string> const &loraWeightsMap,

SpecDecodeDraftingConfig const &draftingConfig,

cudaStream_t stream,

ContextCacheConfig const &contextCacheConfig = {},

std::string const &checkpointDir = "",

std::string const &draftCheckpointDir = ""

)

LLMInferenceRuntime(

std::string const &engineDir,

std::string const &multimodalEngineDir,

std::unordered_map<std::string, std::string> const &loraWeightsMap,

cudaStream_t stream,

ContextCacheConfig const &contextCacheConfig = {},

std::string const &checkpointDir = ""

)

LLMInferenceRuntime(

std::string const &engineDir,

std::string const &multimodalEngineDir,

std::unordered_map<std::string, std::string> const &loraWeightsMap,

ParallelExecutionConfig config

)

LLMInferenceRuntime(

ModelArtifacts &&artifacts,

std::string const &engineDir,

std::string const &multimodalEngineDir,

std::unordered_map<std::string, std::string> const &loraWeightsMap,

std::optional<SpecDecodeDraftingConfig> const &draftingConfig,

cudaStream_t stream,

ContextCacheConfig const &contextCacheConfig = {}

)

~LLMInferenceRuntime() noexcept

bool captureDecodingCUDAGraph(cudaStream_t stream)

bool handleRequest(

LLMGenerationRequest const &request,

LLMGenerationResponse &response,

cudaStream_t stream,

bool outputThinkerEmbeddings = false

)

std::vector<int32_t> countPromptTokens(

LLMGenerationRequest const &request

) const

Return the input size for an explicit text token-count request.

bool genAndSaveSystemPromptKVCache(

std::string const &prompt,

std::string const &loraWeightsName,

cudaStream_t stream

)

void setActionNoiseSeed(int32_t seed)

void setVisualPrunerConfig(VisualPrunerConfig const &config)

Enable visual-token pruning for supported single-device VLM execution.

metrics::LLMPrefillMetrics const &getPrefillMetrics() const

metrics::SpecDecodeGenerationMetrics const &getSpecDecodeGenerationMetrics(

) const

char const *getSpeculativeDecodingStrategyName() const

metrics::LLMGenerationMetrics const &getGenerationMetrics() const

std::optional<ContextCacheMetrics> getContextCacheMetrics() const

metrics::MultimodalMetrics getMultimodalMetrics() const

rt::Tensor const &getEmbeddingTable() const

rt::Tensor const *getBaseModelHiddenStates(int32_t layerIdx) const

int32_t getBaseModelPrefillLength() const

std::vector<std::vector<int32_t>> const &getBaseModelInputTokenIds(

) const

bool hasDraftModel() const

bool ownsGlobalRank(int32_t globalRank) const noexcept

True when this runtime instance owns the requested global rank.

struct ParallelExecutionConfig

Options for constructing the unified runtime with explicit parallel execution settings.

Public Members

ParallelConfig parallelConfig = {}

std::optional<SpecDecodeDraftingConfig> draftingConfig = {}

ContextCacheConfig contextCacheConfig = {}

std::string checkpointDir = {}

std::string draftCheckpointDir = {}

std::vector<int32_t> localRanks = {}

std::unordered_map<int32_t, int32_t> localRankDevices = {}

std::vector<cudaStream_t> localStreams = {}

bool ownsLocalStreams = {true}

std::vector<ParallelBackendHandles> backendHandles = {}

struct ParallelExecutionConfig

Options for constructing the unified runtime with explicit parallel execution settings.

Public Members

ParallelConfig parallelConfig = {}

std::optional<SpecDecodeDraftingConfig> draftingConfig = {}

ContextCacheConfig contextCacheConfig = {}

std::string checkpointDir = {}

std::string draftCheckpointDir = {}

std::vector<int32_t> localRanks = {}

std::unordered_map<int32_t, int32_t> localRankDevices = {}

std::vector<cudaStream_t> localStreams = {}

bool ownsLocalStreams = {true}

std::vector<ParallelBackendHandles> backendHandles = {}