bertweet

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer

Bases: PreTrainedTokenizer

Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`
merges_file	Path to the merges file. TYPE: `str`
normalization	Whether or not to apply a normalization preprocess. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the cls_token. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
mask_token	The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. TYPE: `str`, *optional*, defaults to `"<mask>"` DEFAULT: '<mask>'

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

class BertweetTokenizer(PreTrainedTokenizer):
    """
    Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        normalization (`bool`, *optional*, defaults to `False`):
            Whether or not to apply a normalization preprocess.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
    """
    vocab_files_names = VOCAB_FILES_NAMES

    def __init__(
        self,
        vocab_file,
        merges_file,
        normalization=False,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        **kwargs,
    ):
        """
        Initialize the BertweetTokenizer class with the provided parameters.

        Args:
            self: The instance of the class.
            vocab_file (str): The path to the vocabulary file.
            merges_file (str): The path to the merges file.
            normalization (bool): Flag indicating whether normalization should be applied (default is False).
            bos_token (str): Beginning of sentence token (default is '<s>').
            eos_token (str): End of sentence token (default is '</s>').
            sep_token (str): Separator token (default is '</s>').
            cls_token (str): Class token (default is '<s>').
            unk_token (str): Unknown token (default is '<unk>').
            pad_token (str): Padding token (default is '<pad>').
            mask_token (str): Mask token (default is '<mask>').

        Returns:
            None.

        Raises:
            ImportError: If the 'emoji' library is not installed, a warning is logged, and emoticons or emojis will not be converted to text.
                To resolve this, install emoji library using 'pip3 install emoji==0.6.0'.
            FileNotFoundError: If the vocab_file or merges_file cannot be found or accessed.
            IOError: If there is an issue reading the merges_file with UTF-8 encoding.
            Exception: Any other unforeseen exceptions that might occur during the initialization process.
        """
        try:
            from emoji import demojize

            self.demojizer = demojize
        except ImportError:
            logger.warning(
                "emoji is not installed, thus not converting emoticons or emojis into text. Install emoji: pip3"
                " install emoji==0.6.0"
            )
            self.demojizer = None

        self.vocab_file = vocab_file
        self.merges_file = merges_file

        self.encoder = {}
        self.encoder[str(bos_token)] = 0
        self.encoder[str(pad_token)] = 1
        self.encoder[str(eos_token)] = 2
        self.encoder[str(unk_token)] = 3

        self.add_from_file(vocab_file)

        self.decoder = {v: k for k, v in self.encoder.items()}

        with open(merges_file, encoding="utf-8") as merges_handle:
            merges = merges_handle.read().split("\n")[:-1]
        merges = [tuple(merge.split()[:-1]) for merge in merges]
        self.bpe_ranks = dict(zip(merges, range(len(merges))))
        self.cache = {}

        self.normalization = normalization
        self.tweetPreprocessor = TweetTokenizer()
        self.special_puncts = {"’": "'", "…": "..."}

        super().__init__(
            normalization=normalization,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            **kwargs,
        )

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BERTweet sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s></s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is None:
            return [1] + ([0] * len(token_ids_0)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does
        not make use of token type ids, therefore a list of zeros is returned.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of zeros.
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    @property
    def vocab_size(self):
        """
        Method to retrieve the vocabulary size of the BertweetTokenizer.

        Args:
            self:
                The instance of the BertweetTokenizer class.

                - Type: BertweetTokenizer object.
                - Purpose: Represents the current instance of the BertweetTokenizer class.
                - Restrictions: None.

        Returns:
            The total number of unique tokens in the tokenizer's encoder.

                - Type: int.
                - Purpose: Indicates the vocabulary size of the tokenizer.

        Raises:
            None.
        """
        return len(self.encoder)

    def get_vocab(self):
        """
        Method to retrieve the combined vocabulary from the encoder and added tokens encoder.

        Args:
            self (BertweetTokenizer): An instance of the BertweetTokenizer class.
                Represents the tokenizer object.

        Returns:
            None: Returns a dictionary that combines the encoder and added tokens encoder.
                The keys are word pieces and the values are their corresponding IDs.

        Raises:
            None.
        """
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        """
        This method is part of the BertweetTokenizer class and implements the Byte-Pair Encoding (BPE) algorithm for tokenization.

        Args:
            self (BertweetTokenizer): The instance of the BertweetTokenizer class.
            token (str): The input token to be processed by the BPE algorithm.

        Returns:
            str or None:
                The processed token after applying the BPE algorithm.

                - If the input token is not found in the cache and the BPE algorithm is applied, the processed token is
                returned.
                - If the input token is found in the cache, the corresponding processed token is returned from the cache.
                - If the input token is not found in the cache and the BPE algorithm is not applied, None is returned.

        Raises:
            None: This method does not raise any specific exceptions.
        """
        if token in self.cache:
            return self.cache[token]
        word = tuple(token)
        word = tuple(list(word[:-1]) + [word[-1] + "</w>"])
        pairs = get_pairs(word)

        if not pairs:
            return token

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = "@@ ".join(word)
        word = word[:-4]
        self.cache[token] = word
        return word

    def _tokenize(self, text):
        """Tokenize a string."""
        if self.normalization:  # Perform Tweet normalization before performing BPE
            text = self.normalizeTweet(text)

        split_tokens = []
        words = re.findall(r"\S+\n?", text)
        for token in words:
            split_tokens.extend(list(self.bpe(token).split(" ")))
        return split_tokens

    def normalizeTweet(self, tweet):
        """
        Normalize a raw Tweet
        """
        for punct in self.special_puncts:
            tweet = tweet.replace(punct, self.special_puncts[punct])

        tokens = self.tweetPreprocessor.tokenize(tweet)
        normTweet = " ".join([self.normalizeToken(token) for token in tokens])

        normTweet = (
            normTweet.replace("cannot ", "can not ")
            .replace("n't ", " n't ")
            .replace("n 't ", " n't ")
            .replace("ca n't", "can't")
            .replace("ai n't", "ain't")
        )
        normTweet = (
            normTweet.replace("'m ", " 'm ")
            .replace("'re ", " 're ")
            .replace("'s ", " 's ")
            .replace("'ll ", " 'll ")
            .replace("'d ", " 'd ")
            .replace("'ve ", " 've ")
        )
        normTweet = (
            normTweet.replace(" p . m .", "  p.m.")
            .replace(" p . m ", " p.m ")
            .replace(" a . m .", " a.m.")
            .replace(" a . m ", " a.m ")
        )

        return " ".join(normTweet.split())

    def normalizeToken(self, token):
        """
        Normalize tokens in a Tweet
        """
        lowercased_token = token.lower()
        if token.startswith("@"):
            return "@USER"
        elif lowercased_token.startswith("http") or lowercased_token.startswith("www"):
            return "HTTPURL"
        elif len(token) == 1:
            if token in self.special_puncts:
                return self.special_puncts[token]
            if self.demojizer is not None:
                return self.demojizer(token)
            else:
                return token
        else:
            return token

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.decoder.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        out_string = " ".join(tokens).replace("@@ ", "").strip()
        return out_string

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Saves the vocabulary files required for the BertweetTokenizer.

        Args:
            self (BertweetTokenizer): An instance of the BertweetTokenizer class.
            save_directory (str): The directory where the vocabulary files will be saved.
            filename_prefix (Optional[str], optional): A prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the paths of the saved vocabulary files: out_vocab_file and out_merge_file.

        Raises:
            OSError: If the provided save_directory is not a valid directory.
            FileNotFoundError: If the self.vocab_file does not exist.
            PermissionError: If there is a permission error while copying the vocabulary files.

        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        out_merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        if os.path.abspath(self.merges_file) != os.path.abspath(out_merge_file):
            copyfile(self.merges_file, out_merge_file)

        return out_vocab_file, out_merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)

    def add_from_file(self, f):
        """
        Loads a pre-existing dictionary from a text file and adds its symbols to this instance.
        """
        if isinstance(f, str):
            try:
                with open(f, "r", encoding="utf-8") as fd:
                    self.add_from_file(fd)
            except FileNotFoundError as fnfe:
                raise fnfe
            except UnicodeError:
                raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset")
            return

        lines = f.readlines()
        for lineTmp in lines:
            line = lineTmp.strip()
            idx = line.rfind(" ")
            if idx == -1:
                raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'")
            word = line[:idx]
            self.encoder[word] = len(self.encoder)

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.vocab_size property

Method to retrieve the vocabulary size of the BertweetTokenizer.

PARAMETER	DESCRIPTION
self	The instance of the BertweetTokenizer class. Type: BertweetTokenizer object. Purpose: Represents the current instance of the BertweetTokenizer class. Restrictions: None.

RETURNS	DESCRIPTION
	The total number of unique tokens in the tokenizer's encoder. Type: int. Purpose: Indicates the vocabulary size of the tokenizer.

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.__init__(vocab_file, merges_file, normalization=False, bos_token='<s>', eos_token='</s>', sep_token='</s>', cls_token='<s>', unk_token='<unk>', pad_token='<pad>', mask_token='<mask>', **kwargs)

Initialize the BertweetTokenizer class with the provided parameters.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The path to the vocabulary file. TYPE: str
merges_file	The path to the merges file. TYPE: str
normalization	Flag indicating whether normalization should be applied (default is False). TYPE: bool DEFAULT: False
bos_token	Beginning of sentence token (default is ''). TYPE: str DEFAULT: '<s>'
eos_token	End of sentence token (default is ''). TYPE: str DEFAULT: '</s>'
sep_token	Separator token (default is ''). TYPE: str DEFAULT: '</s>'
cls_token	Class token (default is ''). TYPE: str DEFAULT: '<s>'
unk_token	Unknown token (default is ''). TYPE: str DEFAULT: '<unk>'
pad_token	Padding token (default is ''). TYPE: str DEFAULT: '<pad>'
mask_token	Mask token (default is ''). TYPE: str DEFAULT: '<mask>'

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ImportError	If the 'emoji' library is not installed, a warning is logged, and emoticons or emojis will not be converted to text. To resolve this, install emoji library using 'pip3 install emoji==0.6.0'.
FileNotFoundError	If the vocab_file or merges_file cannot be found or accessed.
IOError	If there is an issue reading the merges_file with UTF-8 encoding.
Exception	Any other unforeseen exceptions that might occur during the initialization process.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def __init__(
    self,
    vocab_file,
    merges_file,
    normalization=False,
    bos_token="<s>",
    eos_token="</s>",
    sep_token="</s>",
    cls_token="<s>",
    unk_token="<unk>",
    pad_token="<pad>",
    mask_token="<mask>",
    **kwargs,
):
    """
    Initialize the BertweetTokenizer class with the provided parameters.

    Args:
        self: The instance of the class.
        vocab_file (str): The path to the vocabulary file.
        merges_file (str): The path to the merges file.
        normalization (bool): Flag indicating whether normalization should be applied (default is False).
        bos_token (str): Beginning of sentence token (default is '<s>').
        eos_token (str): End of sentence token (default is '</s>').
        sep_token (str): Separator token (default is '</s>').
        cls_token (str): Class token (default is '<s>').
        unk_token (str): Unknown token (default is '<unk>').
        pad_token (str): Padding token (default is '<pad>').
        mask_token (str): Mask token (default is '<mask>').

    Returns:
        None.

    Raises:
        ImportError: If the 'emoji' library is not installed, a warning is logged, and emoticons or emojis will not be converted to text.
            To resolve this, install emoji library using 'pip3 install emoji==0.6.0'.
        FileNotFoundError: If the vocab_file or merges_file cannot be found or accessed.
        IOError: If there is an issue reading the merges_file with UTF-8 encoding.
        Exception: Any other unforeseen exceptions that might occur during the initialization process.
    """
    try:
        from emoji import demojize

        self.demojizer = demojize
    except ImportError:
        logger.warning(
            "emoji is not installed, thus not converting emoticons or emojis into text. Install emoji: pip3"
            " install emoji==0.6.0"
        )
        self.demojizer = None

    self.vocab_file = vocab_file
    self.merges_file = merges_file

    self.encoder = {}
    self.encoder[str(bos_token)] = 0
    self.encoder[str(pad_token)] = 1
    self.encoder[str(eos_token)] = 2
    self.encoder[str(unk_token)] = 3

    self.add_from_file(vocab_file)

    self.decoder = {v: k for k, v in self.encoder.items()}

    with open(merges_file, encoding="utf-8") as merges_handle:
        merges = merges_handle.read().split("\n")[:-1]
    merges = [tuple(merge.split()[:-1]) for merge in merges]
    self.bpe_ranks = dict(zip(merges, range(len(merges))))
    self.cache = {}

    self.normalization = normalization
    self.tweetPreprocessor = TweetTokenizer()
    self.special_puncts = {"’": "'", "…": "..."}

    super().__init__(
        normalization=normalization,
        bos_token=bos_token,
        eos_token=eos_token,
        sep_token=sep_token,
        cls_token=cls_token,
        unk_token=unk_token,
        pad_token=pad_token,
        mask_token=mask_token,
        **kwargs,
    )

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.add_from_file(f)

Loads a pre-existing dictionary from a text file and adds its symbols to this instance.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def add_from_file(self, f):
    """
    Loads a pre-existing dictionary from a text file and adds its symbols to this instance.
    """
    if isinstance(f, str):
        try:
            with open(f, "r", encoding="utf-8") as fd:
                self.add_from_file(fd)
        except FileNotFoundError as fnfe:
            raise fnfe
        except UnicodeError:
            raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset")
        return

    lines = f.readlines()
    for lineTmp in lines:
        line = lineTmp.strip()
        idx = line.rfind(" ")
        if idx == -1:
            raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'")
        word = line[:idx]
        self.encoder[word] = len(self.encoder)

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.bpe(token)

This method is part of the BertweetTokenizer class and implements the Byte-Pair Encoding (BPE) algorithm for tokenization.

PARAMETER	DESCRIPTION
self	The instance of the BertweetTokenizer class. TYPE: BertweetTokenizer
token	The input token to be processed by the BPE algorithm. TYPE: str

RETURNS	DESCRIPTION
	str or None: The processed token after applying the BPE algorithm. If the input token is not found in the cache and the BPE algorithm is applied, the processed token is returned. If the input token is found in the cache, the corresponding processed token is returned from the cache. If the input token is not found in the cache and the BPE algorithm is not applied, None is returned.

RAISES	DESCRIPTION
None	This method does not raise any specific exceptions.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def bpe(self, token):
    """
    This method is part of the BertweetTokenizer class and implements the Byte-Pair Encoding (BPE) algorithm for tokenization.

    Args:
        self (BertweetTokenizer): The instance of the BertweetTokenizer class.
        token (str): The input token to be processed by the BPE algorithm.

    Returns:
        str or None:
            The processed token after applying the BPE algorithm.

            - If the input token is not found in the cache and the BPE algorithm is applied, the processed token is
            returned.
            - If the input token is found in the cache, the corresponding processed token is returned from the cache.
            - If the input token is not found in the cache and the BPE algorithm is not applied, None is returned.

    Raises:
        None: This method does not raise any specific exceptions.
    """
    if token in self.cache:
        return self.cache[token]
    word = tuple(token)
    word = tuple(list(word[:-1]) + [word[-1] + "</w>"])
    pairs = get_pairs(word)

    if not pairs:
        return token

    while True:
        bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
        if bigram not in self.bpe_ranks:
            break
        first, second = bigram
        new_word = []
        i = 0
        while i < len(word):
            try:
                j = word.index(first, i)
            except ValueError:
                new_word.extend(word[i:])
                break
            else:
                new_word.extend(word[i:j])
                i = j

            if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                new_word.append(first + second)
                i += 2
            else:
                new_word.append(word[i])
                i += 1
        new_word = tuple(new_word)
        word = new_word
        if len(word) == 1:
            break
        else:
            pairs = get_pairs(word)
    word = "@@ ".join(word)
    word = word[:-4]
    self.cache[token] = word
    return word

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERTweet sequence has the following format:

• single sequence: <s> X </s>
• pair of sequences: <s> A </s></s> B </s>

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A BERTweet sequence has the following format:

    - single sequence: `<s> X </s>`
    - pair of sequences: `<s> A </s></s> B </s>`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    if token_ids_1 is None:
        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
    cls = [self.cls_token_id]
    sep = [self.sep_token_id]
    return cls + token_ids_0 + sep + sep + token_ids_1 + sep

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    out_string = " ".join(tokens).replace("@@ ", "").strip()
    return out_string

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does not make use of token type ids, therefore a list of zeros is returned.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of zeros.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does
    not make use of token type ids, therefore a list of zeros is returned.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of zeros.
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]

    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    if token_ids_1 is None:
        return [1] + ([0] * len(token_ids_0)) + [1]
    return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.get_vocab()

Method to retrieve the combined vocabulary from the encoder and added tokens encoder.

PARAMETER	DESCRIPTION
self	An instance of the BertweetTokenizer class. Represents the tokenizer object. TYPE: BertweetTokenizer

RETURNS	DESCRIPTION
None	Returns a dictionary that combines the encoder and added tokens encoder. The keys are word pieces and the values are their corresponding IDs.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def get_vocab(self):
    """
    Method to retrieve the combined vocabulary from the encoder and added tokens encoder.

    Args:
        self (BertweetTokenizer): An instance of the BertweetTokenizer class.
            Represents the tokenizer object.

    Returns:
        None: Returns a dictionary that combines the encoder and added tokens encoder.
            The keys are word pieces and the values are their corresponding IDs.

    Raises:
        None.
    """
    return dict(self.encoder, **self.added_tokens_encoder)

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.normalizeToken(token)

Normalize tokens in a Tweet

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def normalizeToken(self, token):
    """
    Normalize tokens in a Tweet
    """
    lowercased_token = token.lower()
    if token.startswith("@"):
        return "@USER"
    elif lowercased_token.startswith("http") or lowercased_token.startswith("www"):
        return "HTTPURL"
    elif len(token) == 1:
        if token in self.special_puncts:
            return self.special_puncts[token]
        if self.demojizer is not None:
            return self.demojizer(token)
        else:
            return token
    else:
        return token

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.normalizeTweet(tweet)

Normalize a raw Tweet

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def normalizeTweet(self, tweet):
    """
    Normalize a raw Tweet
    """
    for punct in self.special_puncts:
        tweet = tweet.replace(punct, self.special_puncts[punct])

    tokens = self.tweetPreprocessor.tokenize(tweet)
    normTweet = " ".join([self.normalizeToken(token) for token in tokens])

    normTweet = (
        normTweet.replace("cannot ", "can not ")
        .replace("n't ", " n't ")
        .replace("n 't ", " n't ")
        .replace("ca n't", "can't")
        .replace("ai n't", "ain't")
    )
    normTweet = (
        normTweet.replace("'m ", " 'm ")
        .replace("'re ", " 're ")
        .replace("'s ", " 's ")
        .replace("'ll ", " 'll ")
        .replace("'d ", " 'd ")
        .replace("'ve ", " 've ")
    )
    normTweet = (
        normTweet.replace(" p . m .", "  p.m.")
        .replace(" p . m ", " p.m ")
        .replace(" a . m .", " a.m.")
        .replace(" a . m ", " a.m ")
    )

    return " ".join(normTweet.split())

mindnlp.transformers.models.bertweet.tokenization_bertweet.BertweetTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Saves the vocabulary files required for the BertweetTokenizer.

PARAMETER	DESCRIPTION
self	An instance of the BertweetTokenizer class. TYPE: BertweetTokenizer
save_directory	The directory where the vocabulary files will be saved. TYPE: str
filename_prefix	A prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the paths of the saved vocabulary files: out_vocab_file and out_merge_file.

RAISES	DESCRIPTION
OSError	If the provided save_directory is not a valid directory.
FileNotFoundError	If the self.vocab_file does not exist.
PermissionError	If there is a permission error while copying the vocabulary files.

Source code in mindnlp/transformers/models/bertweet/tokenization_bertweet.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Saves the vocabulary files required for the BertweetTokenizer.

    Args:
        self (BertweetTokenizer): An instance of the BertweetTokenizer class.
        save_directory (str): The directory where the vocabulary files will be saved.
        filename_prefix (Optional[str], optional): A prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the paths of the saved vocabulary files: out_vocab_file and out_merge_file.

    Raises:
        OSError: If the provided save_directory is not a valid directory.
        FileNotFoundError: If the self.vocab_file does not exist.
        PermissionError: If there is a permission error while copying the vocabulary files.

    """
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )
    out_merge_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file, out_vocab_file)
    elif not os.path.isfile(self.vocab_file):
        with open(out_vocab_file, "wb") as fi:
            content_spiece_model = self.sp_model.serialized_model_proto()
            fi.write(content_spiece_model)

    if os.path.abspath(self.merges_file) != os.path.abspath(out_merge_file):
        copyfile(self.merges_file, out_merge_file)

    return out_vocab_file, out_merge_file