Runtime Coordinator#
-
class RuntimeCoordinator#
Owns local LLM rank workers and communication groups for a runtime coordinator.
Public Functions
-
~RuntimeCoordinator()#
-
RuntimeCoordinator(RuntimeCoordinator const&) = delete#
-
RuntimeCoordinator &operator=(RuntimeCoordinator const&) = delete#
-
RuntimeCoordinator(RuntimeCoordinator&&) = delete#
-
RuntimeCoordinator &operator=(RuntimeCoordinator&&) = delete#
-
bool captureDecodingCUDAGraph(cudaStream_t stream = nullptr)#
-
bool supportsSteppedExecution() const noexcept#
True when this coordinator can host a runtime session: inline single-rank execution only in this release; a tensor-parallel deployment answers false.
- std::unique_ptr<RuntimeSession> openSession(
- bool enableProfiling,
- cudaStream_t stream,
- SessionOptions options = {}
Open the runtime session on the one rank this coordinator runs inline; see LLMRankRuntime::openSession. A tensor-parallel deployment cannot host one yet (supportsSteppedExecution() answers false) and this is a contract violation there.
- bool dispatchRequest(
- LLMGenerationRequest const &request,
- bool enableProfiling,
- bool outputThinkerEmbeddings = false,
- cudaStream_t stream = nullptr
- std::vector<int32_t> countPromptTokens(
- LLMGenerationRequest const &request
Return the token count produced by the same preparation path used for inference. Only text requests are supported.
-
void setVisualPrunerConfig(VisualPrunerConfig const &config)#
-
LLMGenerationResponse takeRankResponse(int32_t globalRank)#
-
LLMRankRuntime &rootRuntime()#
-
LLMRankRuntime const &rootRuntime() const#
-
bool ownsGlobalRank(int32_t globalRank) const noexcept#
-
bool localRanksSucceeded() const noexcept#
First failed rank or worker, without polling the GPU.
-
inline int32_t worldSize() const noexcept#
Ranks this plan spans. 1 for a single-device plan.
-
struct Config#
Public Members
-
DeploymentOptions deployment#
-
RuntimeResources resources#
-
std::unique_ptr<ModelArtifacts> modelArtifacts#
-
DeploymentOptions deployment#
-
~RuntimeCoordinator()#
-
struct Config
Public Members
-
DeploymentOptions deployment
-
RuntimeResources resources
-
std::unique_ptr<ModelArtifacts> modelArtifacts
-
DeploymentOptions deployment