Runtime Coordinator#

class RuntimeCoordinator#

Owns local LLM rank workers and communication groups for a runtime coordinator.

Public Functions

explicit RuntimeCoordinator(Config config)#
~RuntimeCoordinator()#
RuntimeCoordinator(RuntimeCoordinator const&) = delete#
RuntimeCoordinator &operator=(RuntimeCoordinator const&) = delete#
RuntimeCoordinator(RuntimeCoordinator&&) = delete#
RuntimeCoordinator &operator=(RuntimeCoordinator&&) = delete#
bool captureDecodingCUDAGraph(cudaStream_t stream = nullptr)#
bool supportsSteppedExecution() const noexcept#

True when this coordinator can host a runtime session: inline single-rank execution only in this release; a tensor-parallel deployment answers false.

std::unique_ptr<RuntimeSession> openSession(
bool enableProfiling,
cudaStream_t stream,
SessionOptions options = {}
)#

Open the runtime session on the one rank this coordinator runs inline; see LLMRankRuntime::openSession. A tensor-parallel deployment cannot host one yet (supportsSteppedExecution() answers false) and this is a contract violation there.

bool dispatchRequest(
LLMGenerationRequest const &request,
bool enableProfiling,
bool outputThinkerEmbeddings = false,
cudaStream_t stream = nullptr
)#
std::vector<int32_t> countPromptTokens(
LLMGenerationRequest const &request
) const#

Return the token count produced by the same preparation path used for inference. Only text requests are supported.

void setVisualPrunerConfig(VisualPrunerConfig const &config)#
LLMGenerationResponse takeRankResponse(int32_t globalRank)#
LLMRankRuntime &rootRuntime()#
LLMRankRuntime const &rootRuntime() const#
bool ownsGlobalRank(int32_t globalRank) const noexcept#
bool localRanksSucceeded() const noexcept#
std::string unavailableReason() const#

First failed rank or worker, without polling the GPU.

inline int32_t worldSize() const noexcept#

Ranks this plan spans. 1 for a single-device plan.

struct Config#

Public Members

DeploymentOptions deployment#
RuntimeResources resources#
std::unique_ptr<ModelArtifacts> modelArtifacts#
struct Config

Public Members

DeploymentOptions deployment
RuntimeResources resources
std::unique_ptr<ModelArtifacts> modelArtifacts