Tokenizer Capability

class TokenizerCapability

Subclassed by trt_edgellm::safety::InternVL3TokenizerCapability

Public Functions

virtual ~TokenizerCapability() = default

virtual SafetyStatus declareResources(

TokenizerArtifactSpec const &artifacts,

ProcessingLimits const &limits,

ResourceDeclaration &declaration

) noexcept = 0

virtual SafetyStatus initialize(

TokenizerArtifactSpec const &artifacts,

PreparedResources const &prepared

) noexcept = 0

virtual TokenizationResult encodeMessages(

LLMGenerationRequest::Request const &request

) noexcept = 0

virtual TokenizationResult encodePrompt(TextView prompt) noexcept = 0

virtual ExpandedTokenResult expandImagePlaceholder(

TokenIdView tokens,

std::size_t imageTokenCount

) noexcept = 0

virtual SafetyStatus decodeResponse(

TokenIdView generated,

MutableTextBuffer response,

std::size_t &written

) noexcept = 0

virtual TokenIdView stopTokens() const noexcept = 0

virtual SafetyStatus deinitialize() noexcept = 0

struct TokenizationResult

Public Members

SafetyStatus status = {SafetyStatus::ok()}

TokenIdView tokens = {}

struct ExpandedTokenResult

Public Members

SafetyStatus status = {SafetyStatus::ok()}

TokenIdView tokens = {}

std::size_t imageOffset = {0}