LLM Model Local Data

class ModelLocalDataFactory : public trt_edgellm::safety::SafetyModelLocalDataFactory

Batched FP16 embeddings and contiguous FP16 KV, shared by the initial InternVL LLM stages. The factory owns query metadata; borrowed tensor views remain owned by the Runner adapter.

Public Functions

ModelLocalDataFactory(

ModelResourceSpec spec,

StageResourceSpec prefill,

StageResourceSpec decode

)

virtual SafetyStatus declareResources(

ResourceDeclaration &output

) const noexcept override

virtual SafetyStatus create(

PreparedResources const &resources,

std::unique_ptr<SafetyModelLocalData> &output

) const noexcept override

Construct metadata over prepared storage; transfer object ownership to SafetyRuntime.

struct ModelResourceSpec

Artifact reader supplies these immutable LLM dimensions before resource declaration.

Public Members

ResourceOwnerId owner = {0}

int32_t vocabSize = {0}

int32_t hiddenSize = {0}

int32_t layerCount = {0}

int32_t kvHeadCount = {0}

int32_t headDim = {0}

int32_t padTokenId = {0}

struct StageResourceSpec

Engine-query metadata for one selected stage/profile; no allocated resources.

Public Members

LMStageId stage = {}

int32_t profileId = {-1}

std::vector<RuntimeTensorDesc> bindings = {}

struct StageResources

Public Members

LMStageId stage = {}

int32_t profileId = {-1}

std::vector<RuntimeTensorView const*> bindings = {}

cudaStream_t stream = {nullptr}

struct ModelLocalData : public trt_edgellm::safety::SafetyModelLocalData

Runtime-owned metadata over manager-owned storage. Processing initializes embedding/RoPE payloads and synchronizes Initialization GPU work before SafetyRuntime publishes readiness.

Public Members

ModelResourceSpec spec = {}

StageResources prefill = {}

StageResources decode = {}

std::array<RuntimeMemoryView, static_cast<std::size_t>(ModelMemory::kCOUNT)> storage = {}

int32_t maxPrefillTokens = {0}

int32_t maxContextLength = {0}