Runtime Coordinator

class RuntimeCoordinator

Owns local LLM rank workers and communication groups for a runtime coordinator.

Public Functions

explicit RuntimeCoordinator(Config config)

~RuntimeCoordinator()

RuntimeCoordinator(RuntimeCoordinator const&) = delete

RuntimeCoordinator &operator=(RuntimeCoordinator const&) = delete

RuntimeCoordinator(RuntimeCoordinator&&) = delete

RuntimeCoordinator &operator=(RuntimeCoordinator&&) = delete

bool captureDecodingCUDAGraph(cudaStream_t stream = nullptr)

bool dispatchRequest(

LLMGenerationRequest const &request,

bool enableProfiling,

bool outputThinkerEmbeddings = false,

cudaStream_t stream = nullptr

)

bool genAndSaveSystemPromptKVCache(

std::string const &prompt,

std::string const &loraWeightsName,

cudaStream_t stream = nullptr

)

void setVisualPrunerConfig(VisualPrunerConfig const &config)

LLMGenerationResponse takeRankResponse(int32_t globalRank)

LLMRankRuntime &rootRuntime()

LLMRankRuntime const &rootRuntime() const

bool ownsGlobalRank(int32_t globalRank) const noexcept

bool localRanksSucceeded() const noexcept

struct Config

Public Members

std::string engineDir

std::string multimodalEngineDir

std::unordered_map<std::string, std::string> loraWeightsMap

ParallelConfig parallelConfig

std::optional<SpecDecodeDraftingConfig> draftingConfig

ContextCacheConfig contextCacheConfig

std::string checkpointDir

std::string draftCheckpointDir

std::vector<int32_t> localRanks

std::unordered_map<int32_t, int32_t> localRankDevices

std::vector<cudaStream_t> localStreams

bool ownsLocalStreams = {true}

std::vector<ParallelBackendHandles> backendHandles

std::unique_ptr<ModelArtifacts> modelArtifacts

struct Config

Public Members

std::string engineDir

std::string multimodalEngineDir

std::unordered_map<std::string, std::string> loraWeightsMap

ParallelConfig parallelConfig

std::optional<SpecDecodeDraftingConfig> draftingConfig

ContextCacheConfig contextCacheConfig

std::string checkpointDir

std::string draftCheckpointDir

std::vector<int32_t> localRanks

std::unordered_map<int32_t, int32_t> localRankDevices

std::vector<cudaStream_t> localStreams

bool ownsLocalStreams = {true}

std::vector<ParallelBackendHandles> backendHandles

std::unique_ptr<ModelArtifacts> modelArtifacts