Tokenizer

BioNeMoAMPLIFYTokenizer

Bases: PreTrainedTokenizerFast, IOMixin

Source code in bionemo/amplify/tokenizer.py

class BioNeMoAMPLIFYTokenizer(transformers.PreTrainedTokenizerFast, IOMixin):  # noqa D101
    def __init__(self):
        """A wrapper to make AutoTokenizer serializable for the ESM2 tokenizer."""
        other = transformers.AutoTokenizer.from_pretrained("chandar-lab/AMPLIFY_350M", use_fast=True)
        self.__dict__.update(other.__dict__)

__init__()

A wrapper to make AutoTokenizer serializable for the ESM2 tokenizer.

Source code in bionemo/amplify/tokenizer.py

def __init__(self):
    """A wrapper to make AutoTokenizer serializable for the ESM2 tokenizer."""
    other = transformers.AutoTokenizer.from_pretrained("chandar-lab/AMPLIFY_350M", use_fast=True)
    self.__dict__.update(other.__dict__)