Model Inference#
-
class ModelInferenceRunner#
Model inference uses this narrow Runner interface to select and execute configured stages.
Subclassed by trt_edgellm::safety::SafetyRuntime
Public Functions
-
virtual ~ModelInferenceRunner() = default#
- virtual SafetyStatus runLMStage(
- LMStageId stage,
- std::size_t slot,
- LMStageOutputs &outputs,
- LMStagePrepareCallback prepare,
- void *userData
-
virtual ~ModelInferenceRunner() = default#
-
class ModelInference#
Recipe-selected request orchestration owned by SafetyRuntime.
Subclassed by trt_edgellm::safety::InternVL3VanillaInference
Public Functions
-
virtual ~ModelInference() = default#
- virtual SafetyStatus run(
- ModelInferenceRunner &runner,
- ProcessingRequest const &request,
- TokenIdBuffer output,
- ModelInferenceResult &result
-
virtual ~ModelInference() = default#
-
struct ModelInferenceResult#
Public Members
-
TokenIdView generatedTokens = {}#
-
FinishReason finishReason = {FinishReason::kNotFinished}#
-
int32_t contextLength = {0}#
-
TokenIdView generatedTokens = {}#
- SafetyStatus trt_edgellm::safety::createModelInference(
- ModelInferenceDefinition const &definition,
- llm::ModelLocalData const &data,
- std::vector<LMStageContext> const &contexts,
- llm::RequestOperations &llmOperations,
- vlm::RequestOperations &visualOperations,
- std::unique_ptr<ModelInference> &output
Construct request orchestration for one admitted model recipe.