Byte Level Bpe Tokenizer

class ByteLevelBpeTokenizer

Public Functions

SafetyStatus declareResources(

std::filesystem::path const &artifact,

ProcessingLimits const &limits,

ByteLevelBpeTokenSpec const *requiredTokens,

std::size_t requiredTokenCount,

ResourceDeclaration &declaration

) noexcept

SafetyStatus initialize(PreparedResources const &prepared) noexcept

SafetyStatus encode(

TextView text,

int32_t *output,

std::size_t capacity,

std::size_t &outputSize

) noexcept

SafetyStatus decode(

TokenIdView tokens,

TokenIdView stopTokens,

MutableTextBuffer output,

std::size_t &outputSize

) const noexcept

int32_t specialTokenId(std::string_view token) const noexcept

bool hasTokenId(int32_t id) const noexcept

SafetyStatus deinitialize() noexcept

struct ByteLevelBpeTokenSpec

Public Members

std::string_view bytes = {}

int32_t id = {-1}

bool special = {false}