tokenization_utils

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer

Bases: PreTrainedTokenizerBase

Base class for all slow tokenizers.

Inherits from [~tokenization_utils_base.PreTrainedTokenizerBase].

Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary.

This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).

Source code in mindnlp/transformers/tokenization_utils.py

class PreTrainedTokenizer(PreTrainedTokenizerBase):
    """
    Base class for all slow tokenizers.

    Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].

    Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
    pretrained tokenizers as well as adding tokens to the vocabulary.

    This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
    specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
    """
    def __init__(self, **kwargs):
        """
        Initialize the PreTrainedTokenizer class.

        Args:
            self: An instance of the PreTrainedTokenizer class.

        Returns:
            None.

        Raises:
            None.
        """
        # 1. Init the parent class

        self.tokens_trie = Trie()

        # 2. init `_added_tokens_decoder` if child class did not
        if not hasattr(self, "_added_tokens_decoder"):
            self._added_tokens_decoder: Dict[int, AddedToken] = {}

        # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
        self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))
        self._added_tokens_encoder: Dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}

        # 4 init the parent class
        super().__init__(**kwargs)

        # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
        # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`
        self._add_tokens(
            [token for token in self.all_special_tokens_extended if token not in self._added_tokens_encoder],
            special_tokens=True,
        )

        self._decode_use_source_tokenizer = False

    @property
    def is_fast(self) -> bool:
        """
        Checks if the tokenizer is fast.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.

        Returns:
            bool: A boolean value indicating whether the tokenizer is fast or not.

        Raises:
            None.

        This method returns True if the tokenizer is fast, and False otherwise.

        The tokenizer's speed is determined by various factors, such as its implementation, data size,
        and computational resources available.

        This information can be useful for optimizing the tokenization process or making decisions based
        on the tokenizer's performance.
        """
        return False

    @property
    def vocab_size(self) -> int:
        """
        `int`: Size of the base vocabulary (without the added tokens).
        """
        raise NotImplementedError

    @property
    def added_tokens_encoder(self) -> Dict[str, int]:
        """
        Returns the sorted mapping from string to index. The added tokens encoder is cached for performance
        optimisation in `self._added_tokens_encoder` for the slow tokenizers.
        """
        return {k.content: v for v, k in sorted(self._added_tokens_decoder.items(), key=lambda item: item[0])}

    @property
    def added_tokens_decoder(self) -> Dict[int, AddedToken]:
        """
        Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.

        Returns:
            `Dict[str, int]`: The added tokens.
        """
        return dict(sorted(self._added_tokens_decoder.items(), key=lambda item: item[0]))

    @added_tokens_decoder.setter
    def added_tokens_decoder(self, value: Dict[int, Union[AddedToken, str]]) -> Dict[int, AddedToken]:
        """
        Decodes added tokens and updates the internal state of the tokenizer.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
            value (Dict[int, Union[AddedToken, str]]): A dictionary containing the mapping of indices to added tokens or strings.
                The keys represent the indices and the values can be either AddedToken objects or strings.

        Returns:
            Dict[int, AddedToken]: A dictionary containing the decoded added tokens. The keys represent the indices and the
                values are AddedToken objects.

        Raises:
            ValueError: If the provided `value` is not a dictionary, or if any of its elements have incorrect types. The
                correct format for each element should be a valid integer index mapping to an AddedToken object or a string.
        """
        # Always raise an error if string because users should define the behavior
        for index, token in value.items():
            if not isinstance(token, (str, AddedToken)) or not isinstance(index, int):
                raise ValueError(
                    f"The provided `added_tokens_decoder` has an element of type {index.__class__, token.__class__}, should be a dict of {int, Union[AddedToken, str]}"
                )

            self._added_tokens_decoder[index] = AddedToken(token) if isinstance(token, str) else token
            self._added_tokens_encoder[str(token)] = index

    def get_added_vocab(self) -> Dict[str, int]:
        """
        Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from
        the fast call because for now we always add the tokens even if they are already in the vocabulary. This is
        something we should change.

        Returns:
            `Dict[str, int]`: The added tokens.
        """
        return self._added_tokens_encoder

    def __len__(self):
        """
        Size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if
        there is a hole in the vocab, we will add tokenizers at a wrong index.
        """
        return len(set(self.get_vocab().keys()))

    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
        """
        Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to
        it with indices starting from length of the current vocabulary. Special tokens are sometimes already in the
        vocab which is why they have to be handled specifically.

        Args:
            new_tokens (`List[str]`or `List[tokenizers.AddedToken]`):
                Token(s) to add in vocabulary. A token is counted as added if it's not already in the vocabulary
                (tested by checking if the tokenizer assign the index of the `unk_token` to them). If a token is part
                of the vocabulary then we simply mark this token as an `AddedToken` which allows to control the
                stripping and normalization of this token. This is NOT possible in `tokenizers`.
            special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the tokens should be added as special tokens.

        Returns:
            `int`: The number of tokens actually added to the vocabulary.

        Example:
            ```python
            >>> # Let's see how to increase the vocabulary of Bert model and tokenizer
            >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
            >>> model = BertModel.from_pretrained("bert-base-uncased")
            ...
            >>> num_added_toks = tokenizer.add_tokens(["new_tok1", "my_new-tok2"])
            >>> print("We have added", num_added_toks, "tokens")
            >>> # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer.
            >>> model.resize_token_embeddings(len(tokenizer))
            ```
        """
        added_tokens = 0
        if new_tokens is None:
            return added_tokens
        # TODO this is fairly slow to improve!
        current_vocab = self.get_vocab().copy()
        new_idx = len(current_vocab)  # only call this once, len gives the last index + 1
        for token in new_tokens:
            if not isinstance(token, (str, AddedToken)):
                raise TypeError(f"Token {token} is not a string but a {type(token)}.")
            if str(token) == "":
                continue
            if isinstance(token, str):
                if token in self._added_tokens_encoder:
                    continue
                # very important for fast and slow equivalence!
                is_special = token in self.all_special_tokens or special_tokens
                token = AddedToken(
                    token, rstrip=False, lstrip=False, normalized=not is_special, special=is_special
                )
            elif special_tokens:
                # doing token.special=True changes the normalization! will fix in rust
                # this is important and the only reason why the AddedTokens in each class are normalized by default
                token.__setstate__({"special": True, "normalized": token.normalized})
            if token in self._added_tokens_decoder:
                continue
            if not token.special and token.normalized and getattr(self, "do_lower_case", False):
                # Normalize if requested
                token.content = token.content.lower()
            if token.content not in current_vocab:
                token_index = new_idx + added_tokens
                current_vocab[token.content] = token_index
                added_tokens += 1
            else:
                token_index = current_vocab[token.content]

            if token.special and str(token) not in self.all_special_tokens:
                self._additional_special_tokens.append(token)
            # the setter automatically updates the reverse map
            self._added_tokens_decoder[token_index] = token
            self._added_tokens_encoder[token.content] = token_index
            if self.verbose:
                logger.info(f"Adding {token} to the vocabulary")

        self._update_trie()
        return added_tokens

    def _update_trie(self, unique_no_split_tokens: Optional[str] = []):
        """
        Updates the tokens trie with unique tokens and unique_no_split_tokens.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
            unique_no_split_tokens (Optional[str]):
                A list of unique tokens to be added to the tokens trie. Default is an empty list.

        Returns:
            None.

        Raises:
            TypeError: If the unique_no_split_tokens parameter is not a list of strings.
            ValueError: If the unique_no_split_tokens list contains non-string elements.
        """
        for token in self._added_tokens_decoder.values():
            if token not in self.tokens_trie._tokens:
                self.tokens_trie.add(token.content)
        for token in unique_no_split_tokens:
            if token not in self.tokens_trie._tokens:
                self.tokens_trie.add(token)

    def num_special_tokens_to_add(self, pair: bool = False) -> int:
        """
        Returns the number of added tokens when encoding a sequence with special tokens.

        <Tip>

        This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
        this inside your training loop.

        </Tip>

        Args:
            pair (`bool`, *optional*, defaults to `False`):
                Whether the number of added tokens should be computed in the case of a sequence pair or a single
                sequence.

        Returns:
            `int`: Number of special tokens added to sequences.
        """
        token_ids_0 = []
        token_ids_1 = []
        return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None))

    def tokenize(self, text: TextInput, **kwargs) -> List[str]:
        """
        Converts a string in a sequence of tokens, using the tokenizer.

        Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
        (BPE/SentencePieces/WordPieces). Takes care of added tokens.

        Args:
            text (`str`):
                The sequence to be encoded.
            **kwargs (additional keyword arguments):
                Passed along to the model-specific `prepare_for_tokenization` preprocessing method.

        Returns:
            `List[str]`: The list of tokens.
        """
        split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)

        text, kwargs = self.prepare_for_tokenization(text, **kwargs)

        if kwargs:
            logger.warning(f"Keyword arguments {kwargs} not recognized.")

        if hasattr(self, "do_lower_case") and self.do_lower_case:
            # convert non-special tokens to lowercase. Might be super slow as well?
            escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
            escaped_special_toks += [
                re.escape(s_tok.content)
                for s_tok in (self._added_tokens_decoder.values())
                if not s_tok.special and s_tok.normalized
            ]
            pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
            text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)

        if split_special_tokens:
            no_split_token = []
            tokens = [text]
        else:
            no_split_token = self._added_tokens_encoder.keys()  # don't split on any of the added tokens
            # "This is something<special_token_1>  else"
            tokens = self.tokens_trie.split(text)

        # ["This is something", "<special_token_1>", "  else"]
        for i, token in enumerate(tokens):
            if token in no_split_token:
                tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)
                left = tokens[i - 1] if i > 0 else None
                right = tokens[i + 1] if i < len(tokens) - 1 else None
                if isinstance(tok_extended, AddedToken):
                    if tok_extended.rstrip and right:
                        # A bit counter-intuitive but we strip the left of the string
                        # since tok_extended.rstrip means the special token is eating all white spaces on its right
                        tokens[i + 1] = right.lstrip()
                    # Strip white spaces on the left
                    if tok_extended.lstrip and left:
                        tokens[i - 1] = left.rstrip()  # Opposite here
                    if tok_extended.single_word and left and left[-1] != " ":
                        tokens[i - 1] += token
                        tokens[i] = ""
                    elif tok_extended.single_word and right and right[0] != " ":
                        tokens[i + 1] = token + tokens[i + 1]
                        tokens[i] = ""
                else:
                    raise ValueError(
                        f"{tok_extended} cannot be tokenized because it was not properly added"
                        f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"
                    )
        # ["This is something", "<special_token_1>", "else"]
        tokenized_text = []
        for token in tokens:
            # Need to skip eventual empty (fully stripped) tokens
            if not token:
                continue
            if token in no_split_token:
                tokenized_text.append(token)
            else:
                tokenized_text.extend(self._tokenize(token))
        # ["This", " is", " something", "<special_token_1>", "else"]
        return tokenized_text

    def _tokenize(self, text, **kwargs):
        """
        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).

        Do NOT take care of added tokens.
        """
        raise NotImplementedError

    def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
        """
        Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
        vocabulary.

        Args:
            tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).

        Returns:
            `int` or `List[int]`: The token id or list of token ids.
        """
        if tokens is None:
            return None

        if isinstance(tokens, str):
            return self._convert_token_to_id_with_added_voc(tokens)

        ids = []
        for token in tokens:
            ids.append(self._convert_token_to_id_with_added_voc(token))
        return ids

    def _convert_token_to_id_with_added_voc(self, token):
        """
        Method _convert_token_to_id_with_added_voc in the PreTrainedTokenizer class.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
            token (str): The token to be converted to an ID. It can be any valid string token.

        Returns:
            None:
                If the token is None, it returns None. Otherwise, it returns the ID associated with the token.

        Raises:
            None.
        """
        if token is None:
            return None

        if token in self._added_tokens_encoder:
            return self._added_tokens_encoder[token]
        return self._convert_token_to_id(token)

    def _convert_token_to_id(self, token):
        """
        Converts a token to its corresponding ID.

        Args:
            self (PreTrainedTokenizer): An instance of the PreTrainedTokenizer class.
            token (str): The token to be converted to ID.

        Returns:
            None.

        Raises:
            NotImplementedError: If the method is not implemented or overridden by a subclass.

        """
        raise NotImplementedError

    def _encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput, EncodedInput],
        text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        """
        Method to encode text inputs for a transformer model.

        Args:
            self: The instance of the PreTrainedTokenizer class.
            text (Union[TextInput, PreTokenizedInput, EncodedInput]):
                The input text to encode. It can be a string, a list/tuple of strings, or a list/tuple of integers.
            text_pair (Optional[Union[TextInput, PreTokenizedInput, EncodedInput]]):
                Optional second input text to encode. It follows the same format as 'text'.
            add_special_tokens (bool): Whether to add special tokens for the model. Default is True.
            padding_strategy (PaddingStrategy): The padding strategy to apply. Default is PaddingStrategy.DO_NOT_PAD.
            truncation_strategy (TruncationStrategy):
                The truncation strategy to apply. Default is TruncationStrategy.DO_NOT_TRUNCATE.
            max_length (Optional[int]): The maximum length of the encoded inputs.
            stride (int): The stride to use when overflowing tokens. Default is 0.
            is_split_into_words (bool): Whether the input is split into words. Default is False.
            pad_to_multiple_of (Optional[int]): Pad to a multiple of this value.
            return_tensors (Optional[Union[str, TensorType]]): The type of tensors to return.
            return_token_type_ids (Optional[bool]): Whether to return token type IDs.
            return_attention_mask (Optional[bool]): Whether to return attention masks.
            return_overflowing_tokens (bool): Whether to return overflowing tokens.
            return_special_tokens_mask (bool): Whether to return a special tokens mask.
            return_offsets_mapping (bool): Whether to return offsets mapping. Not available for Python tokenizers.
            return_length (bool): Whether to return the length of the input sequence.
            verbose (bool): Whether to output verbose information.

        Returns:
            BatchEncoding: A dictionary containing the encoded inputs ready to be fed into the transformer model.

        Raises:
            NotImplementedError: If 'return_offsets_mapping' is set to True, as this feature is not available for Python tokenizers.
            ValueError: If the input 'text' is invalid based on the specified conditions.
        """
        def get_input_ids(text):
            if isinstance(text, str):
                tokens = self.tokenize(text, **kwargs)
                return self.convert_tokens_to_ids(tokens)
            if isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):
                if is_split_into_words:
                    tokens = list(
                        itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))
                    )
                    return self.convert_tokens_to_ids(tokens)
                return self.convert_tokens_to_ids(text)
            if isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
                return text
            if is_split_into_words:
                raise ValueError(
                    f"Input {text} is not valid. Should be a string or a list/tuple of strings when"
                    " `is_split_into_words=True`."
                )
            raise ValueError(
                f"Input {text} is not valid. Should be a string, a list/tuple of strings or a list/tuple of"
                " integers."
            )

        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast. "
                "More information on available tokenizers at "
                "https://github.com/huggingface/transformers/pull/2674"
            )

        first_ids = get_input_ids(text)
        second_ids = get_input_ids(text_pair) if text_pair is not None else None

        return self.prepare_for_model(
            first_ids,
            pair_ids=second_ids,
            add_special_tokens=add_special_tokens,
            padding=padding_strategy.value,
            truncation=truncation_strategy.value,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            prepend_batch_axis=True,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            verbose=verbose,
        )

    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
            List[PreTokenizedInputPair],
            List[EncodedInput],
            List[EncodedInputPair],
        ],
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        """
        This method '_batch_encode_plus' in the class 'PreTrainedTokenizer' performs batch encoding of text or text pairs.

        Args:
            self: The instance of the class.
            batch_text_or_text_pairs (List): A list of text input or text input pairs to be encoded.
            add_special_tokens (bool): Whether to add special tokens during encoding. Default is True.
            padding_strategy (PaddingStrategy):
                The padding strategy to use during encoding. Default is PaddingStrategy.DO_NOT_PAD.
            truncation_strategy (TruncationStrategy):
                The truncation strategy to use during encoding. Default is TruncationStrategy.DO_NOT_TRUNCATE.
            max_length (Optional[int]): The maximum length of the encoded sequence. Default is None.
            stride (int): The stride for splitting inputs into multiple sequences. Default is 0.
            is_split_into_words (bool): Whether the input text is already split into words. Default is False.
            pad_to_multiple_of (Optional[int]): Pad the sequence length to be a multiple of this value. Default is None.
            return_tensors (Optional[Union[str, TensorType]]): The type of tensor to return. Default is None.
            return_token_type_ids (Optional[bool]): Whether to return token type ids. Default is None.
            return_attention_mask (Optional[bool]): Whether to return attention masks. Default is None.
            return_overflowing_tokens (bool): Whether to return overflowing tokens. Default is False.
            return_special_tokens_mask (bool): Whether to return special tokens mask. Default is False.
            return_offsets_mapping (bool): Whether to return offsets mapping. Default is False.
            return_length (bool): Whether to return the length of the encoded sequence. Default is False.
            verbose (bool): Whether to print verbose output. Default is True.
            kwargs: Additional keyword arguments.

        Returns:
            BatchEncoding: The encoded batch of inputs as a BatchEncoding object.

        Raises:
            ValueError: If the input is not valid, should be a string, a list/tuple of strings, or a list/tuple of integers.
            NotImplementedError: If 'return_offset_mapping' is requested, which is not available when using Python tokenizers.
                To use this feature, change the tokenizer to one deriving from transformers.PreTrainedTokenizerFast.
        """
        def get_input_ids(text):
            if isinstance(text, str):
                tokens = self.tokenize(text, **kwargs)
                return self.convert_tokens_to_ids(tokens)
            if isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str):
                if is_split_into_words:
                    tokens = list(
                        itertools.chain(*(self.tokenize(t, is_split_into_words=True, **kwargs) for t in text))
                    )
                    return self.convert_tokens_to_ids(tokens)
                return self.convert_tokens_to_ids(text)
            if isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
                return text
            raise ValueError(
                "Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers."
            )

        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast."
            )

        input_ids = []
        for ids_or_pair_ids in batch_text_or_text_pairs:
            if not isinstance(ids_or_pair_ids, (list, tuple)):
                ids, pair_ids = ids_or_pair_ids, None
            elif is_split_into_words and not isinstance(ids_or_pair_ids[0], (list, tuple)):
                ids, pair_ids = ids_or_pair_ids, None
            else:
                ids, pair_ids = ids_or_pair_ids

            first_ids = get_input_ids(ids)
            second_ids = get_input_ids(pair_ids) if pair_ids is not None else None
            input_ids.append((first_ids, second_ids))

        batch_outputs = self._batch_prepare_for_model(
            input_ids,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            return_tensors=return_tensors,
            verbose=verbose,
        )

        return BatchEncoding(batch_outputs)

    def _batch_prepare_for_model(
        self,
        batch_ids_pairs: List[Union[PreTokenizedInputPair, Tuple[List[int], None]]],
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[str] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_length: bool = False,
        verbose: bool = True,
    ) -> BatchEncoding:
        """
        Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
        adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
        manages a moving window (with user defined stride) for overflowing tokens

        Args:
            batch_ids_pairs: list of tokenized input ids or input ids pairs
        """
        batch_outputs = {}
        for first_ids, second_ids in batch_ids_pairs:
            outputs = self.prepare_for_model(
                first_ids,
                second_ids,
                add_special_tokens=add_special_tokens,
                padding=PaddingStrategy.DO_NOT_PAD.value,  # we pad in batch afterward
                truncation=truncation_strategy.value,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=None,  # we pad in batch afterward
                return_attention_mask=False,  # we pad in batch afterward
                return_token_type_ids=return_token_type_ids,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_length=return_length,
                return_tensors=None,  # We convert the whole batch to tensors at the end
                prepend_batch_axis=False,
                verbose=verbose,
            )

            for key, value in outputs.items():
                if key not in batch_outputs:
                    batch_outputs[key] = []
                batch_outputs[key].append(value)

        batch_outputs = self.pad(
            batch_outputs,
            padding=padding_strategy.value,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
        )

        batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)

        return batch_outputs

    def prepare_for_tokenization(
        self, text: str, is_split_into_words: bool = False, **kwargs
    ) -> Tuple[str, Dict[str, Any]]:
        """
        Performs any necessary transformations before tokenization.

        This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the
        `kwargs` at the end of the encoding process to be sure all the arguments have been used.

        Args:
            text (`str`):
                The text to prepare.
            is_split_into_words (`bool`, *optional*, defaults to `False`):
                Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the
                tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)
                which it will tokenize. This is useful for NER or token classification.
            kwargs (`Dict[str, Any]`, *optional*):
                Keyword arguments to use for the tokenization.

        Returns:
            `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs.
        """
        return (text, kwargs)

    def get_special_tokens_mask(
        self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.

        Args:
            token_ids_0 (`List[int]`):
                List of ids of the first sequence.
            token_ids_1 (`List[int]`, *optional*):
                List of ids of the second sequence.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            if token_ids_1 is not None:
                raise ValueError(
                    "You should not supply a second sequence if the provided sequence of "
                    "ids is already formatted with special tokens for the model."
                )

            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )
        return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))

    @overload
    def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str:
        """
        Converts a list of token IDs to their corresponding tokens in the original text.

        Args:
            self (PreTrainedTokenizer): An instance of the PreTrainedTokenizer class.
            ids (int): The list of token IDs to be converted.
            skip_special_tokens (bool, optional): Whether to skip special tokens during the conversion.
                Defaults to False.

        Returns:
            str: The reforwarded text from the given token IDs.

        Raises:
            None.

        Note:
            - Special tokens refer to tokens like [CLS], [SEP], [PAD], etc.
            - If skip_special_tokens is set to True, the special tokens will be excluded from the returned text.

        Example:
            ```python
            >>> tokenizer = PreTrainedTokenizer()
            >>> token_ids = [101, 202, 303, 102]
            >>> text = tokenizer.convert_ids_to_tokens(token_ids)
            >>> # text now contains "[CLS] token1 token2 [SEP]"
            ```
        """

    @overload
    def convert_ids_to_tokens(self, ids: List[int], skip_special_tokens: bool = False) -> List[str]:
        """
        Converts a sequence of token ids into a sequence of tokens.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
            ids (List[int]): The list of token ids to be converted into tokens.
            skip_special_tokens (bool, optional): Whether to skip special tokens during conversion. Defaults to False.

        Returns:
            List[str]: A list of tokens corresponding to the input token ids.

        Raises:
            None.
        """

    def convert_ids_to_tokens(
        self, ids: Union[int, List[int]], skip_special_tokens: bool = False
    ) -> Union[str, List[str]]:
        """
        Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
        added tokens.

        Args:
            ids (`int` or `List[int]`):
                The token id (or token ids) to convert to tokens.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.

        Returns:
            `str` or `List[str]`: The decoded token(s).
        """
        if isinstance(ids, int):
            if ids in self._added_tokens_decoder:
                return self._added_tokens_decoder[ids].content
            return self._convert_id_to_token(ids)
        tokens = []
        for index in ids:
            index = int(index)
            if skip_special_tokens and index in self.all_special_ids:
                continue
            if index in self._added_tokens_decoder:
                tokens.append(self._added_tokens_decoder[index].content)
            else:
                tokens.append(self._convert_id_to_token(index))
        return tokens

    def _convert_id_to_token(self, index: int) -> str:
        """
        Converts an index to its corresponding token in the tokenizer's vocabulary.

        Args:
            self (PreTrainedTokenizer): An instance of the PreTrainedTokenizer class.
            index (int): The index of the token in the vocabulary.

        Returns:
            str: The token corresponding to the given index.

        Raises:
            NotImplementedError: If the method has not been implemented by a subclass.
        """
        raise NotImplementedError

    def convert_tokens_to_string(self, tokens: List[str]) -> str:
        """
        Converts a list of tokens into a single string.

        Args:
            self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
            tokens (List[str]): A list of string tokens to be converted into a single string. 

        Returns:
            str: A string representing the concatenated tokens.

        Raises:
            None
        """
        return " ".join(tokens)

    def _decode(
        self,
        token_ids: List[int],
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        spaces_between_special_tokens: bool = True,
        **kwargs,
    ) -> str:
        """
        This method decodes the given token IDs into a string representation.

        Args:
            self (object): The instance of the PreTrainedTokenizer class.
            token_ids (List[int]): A list of token IDs to be decoded into a string.
            skip_special_tokens (bool): A flag indicating whether special tokens should be skipped during decoding.
                Defaults to False.
            clean_up_tokenization_spaces (bool): A flag indicating whether to clean up tokenization spaces.
                If None, the value of self.clean_up_tokenization_spaces is used. Defaults to None.
            spaces_between_special_tokens (bool): A flag indicating whether spaces should be added between special
                tokens in the decoded text. Defaults to True.

        Returns:
            str: The decoded string representation of the input token IDs.

        Raises:
            None
        """
        self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)

        filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
        legacy_added_tokens = set(self._added_tokens_encoder.keys()) - set(self.all_special_tokens) | {
            token for token in self.additional_special_tokens if self.convert_tokens_to_ids(token) >= self.vocab_size
        }
        # To avoid mixing byte-level and unicode for byte-level BPT
        # we need to build string separately for added tokens and byte-level tokens
        # cf. https://github.com/huggingface/transformers/issues/1133
        sub_texts = []
        current_sub_text = []
        # TODO @ArthurZ in version 5, special tokens should be handled in convert_tokens_to_string, while _convert_tokens_to_string
        for token in filtered_tokens:
            if skip_special_tokens and token in self.all_special_ids:
                continue
            if token in legacy_added_tokens:
                if current_sub_text:
                    string = self.convert_tokens_to_string(current_sub_text)
                    if len(string) > 0:
                        sub_texts.append(string)
                    current_sub_text = []
                sub_texts.append(token)
            else:
                current_sub_text.append(token)
        if current_sub_text:
            sub_texts.append(self.convert_tokens_to_string(current_sub_text))

        if spaces_between_special_tokens:
            text = " ".join(sub_texts)
        else:
            text = "".join(sub_texts)

        clean_up_tokenization_spaces = (
            clean_up_tokenization_spaces
            if clean_up_tokenization_spaces is not None
            else self.clean_up_tokenization_spaces
        )
        if clean_up_tokenization_spaces:
            clean_text = self.clean_up_tokenization(text)
            return clean_text
        return text

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.added_tokens_decoder: Dict[int, AddedToken] property writable

Returns the added tokens in the vocabulary as a dictionary of index to AddedToken.

RETURNS	DESCRIPTION
Dict[int, AddedToken]	Dict[str, int]: The added tokens.

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.added_tokens_encoder: Dict[str, int] property

Returns the sorted mapping from string to index. The added tokens encoder is cached for performance optimisation in self._added_tokens_encoder for the slow tokenizers.

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.is_fast: bool property

Checks if the tokenizer is fast.

PARAMETER	DESCRIPTION
self	The instance of the PreTrainedTokenizer class. TYPE: PreTrainedTokenizer

RETURNS	DESCRIPTION
bool	A boolean value indicating whether the tokenizer is fast or not. TYPE: bool

This method returns True if the tokenizer is fast, and False otherwise.

The tokenizer's speed is determined by various factors, such as its implementation, data size, and computational resources available.

This information can be useful for optimizing the tokenization process or making decisions based on the tokenizer's performance.

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.vocab_size: int property

int: Size of the base vocabulary (without the added tokens).

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.__init__(**kwargs)

Initialize the PreTrainedTokenizer class.

PARAMETER	DESCRIPTION
self	An instance of the PreTrainedTokenizer class.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/tokenization_utils.py

def __init__(self, **kwargs):
    """
    Initialize the PreTrainedTokenizer class.

    Args:
        self: An instance of the PreTrainedTokenizer class.

    Returns:
        None.

    Raises:
        None.
    """
    # 1. Init the parent class

    self.tokens_trie = Trie()

    # 2. init `_added_tokens_decoder` if child class did not
    if not hasattr(self, "_added_tokens_decoder"):
        self._added_tokens_decoder: Dict[int, AddedToken] = {}

    # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
    self._added_tokens_decoder.update(kwargs.pop("added_tokens_decoder", {}))
    self._added_tokens_encoder: Dict[str, int] = {k.content: v for v, k in self._added_tokens_decoder.items()}

    # 4 init the parent class
    super().__init__(**kwargs)

    # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
    # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`
    self._add_tokens(
        [token for token in self.all_special_tokens_extended if token not in self._added_tokens_encoder],
        special_tokens=True,
    )

    self._decode_use_source_tokenizer = False

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.__len__()

Size of the full vocabulary with the added tokens. Counts the keys and not the values because otherwise if there is a hole in the vocab, we will add tokenizers at a wrong index.

Source code in mindnlp/transformers/tokenization_utils.py

def __len__(self):
    """
    Size of the full vocabulary with the added tokens. Counts the `keys` and not the `values` because otherwise if
    there is a hole in the vocab, we will add tokenizers at a wrong index.
    """
    return len(set(self.get_vocab().keys()))

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.convert_ids_to_tokens(ids, skip_special_tokens=False)

Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens.

PARAMETER	DESCRIPTION
ids	The token id (or token ids) to convert to tokens. TYPE: `int` or `List[int]`
skip_special_tokens	Whether or not to remove special tokens in the decoding. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
Union[str, List[str]]	str or List[str]: The decoded token(s).

Source code in mindnlp/transformers/tokenization_utils.py

def convert_ids_to_tokens(
    self, ids: Union[int, List[int]], skip_special_tokens: bool = False
) -> Union[str, List[str]]:
    """
    Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
    added tokens.

    Args:
        ids (`int` or `List[int]`):
            The token id (or token ids) to convert to tokens.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.

    Returns:
        `str` or `List[str]`: The decoded token(s).
    """
    if isinstance(ids, int):
        if ids in self._added_tokens_decoder:
            return self._added_tokens_decoder[ids].content
        return self._convert_id_to_token(ids)
    tokens = []
    for index in ids:
        index = int(index)
        if skip_special_tokens and index in self.all_special_ids:
            continue
        if index in self._added_tokens_decoder:
            tokens.append(self._added_tokens_decoder[index].content)
        else:
            tokens.append(self._convert_id_to_token(index))
    return tokens

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.convert_tokens_to_ids(tokens)

Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the vocabulary.

PARAMETER	DESCRIPTION
tokens	One or several token(s) to convert to token id(s). TYPE: `str` or `List[str]`

RETURNS	DESCRIPTION
Union[int, List[int]]	int or List[int]: The token id or list of token ids.

Source code in mindnlp/transformers/tokenization_utils.py

def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
    """
    Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
    vocabulary.

    Args:
        tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).

    Returns:
        `int` or `List[int]`: The token id or list of token ids.
    """
    if tokens is None:
        return None

    if isinstance(tokens, str):
        return self._convert_token_to_id_with_added_voc(tokens)

    ids = []
    for token in tokens:
        ids.append(self._convert_token_to_id_with_added_voc(token))
    return ids

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.convert_tokens_to_string(tokens)

Converts a list of tokens into a single string.

PARAMETER	DESCRIPTION
self	The instance of the PreTrainedTokenizer class. TYPE: PreTrainedTokenizer
tokens	A list of string tokens to be converted into a single string. TYPE: List[str]

RETURNS	DESCRIPTION
str	A string representing the concatenated tokens. TYPE: str

Source code in mindnlp/transformers/tokenization_utils.py

def convert_tokens_to_string(self, tokens: List[str]) -> str:
    """
    Converts a list of tokens into a single string.

    Args:
        self (PreTrainedTokenizer): The instance of the PreTrainedTokenizer class.
        tokens (List[str]): A list of string tokens to be converted into a single string. 

    Returns:
        str: A string representing the concatenated tokens.

    Raises:
        None
    """
    return " ".join(tokens)

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.get_added_vocab()

Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from the fast call because for now we always add the tokens even if they are already in the vocabulary. This is something we should change.

RETURNS	DESCRIPTION
Dict[str, int]	Dict[str, int]: The added tokens.

Source code in mindnlp/transformers/tokenization_utils.py

def get_added_vocab(self) -> Dict[str, int]:
    """
    Returns the added tokens in the vocabulary as a dictionary of token to index. Results might be different from
    the fast call because for now we always add the tokens even if they are already in the vocabulary. This is
    something we should change.

    Returns:
        `Dict[str, int]`: The added tokens.
    """
    return self._added_tokens_encoder

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model or encode_plus methods.

PARAMETER	DESCRIPTION
token_ids_0	List of ids of the first sequence. TYPE: `List[int]`
token_ids_1	List of ids of the second sequence. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/tokenization_utils.py

def get_special_tokens_mask(
    self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.

    Args:
        token_ids_0 (`List[int]`):
            List of ids of the first sequence.
        token_ids_1 (`List[int]`, *optional*):
            List of ids of the second sequence.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        if token_ids_1 is not None:
            raise ValueError(
                "You should not supply a second sequence if the provided sequence of "
                "ids is already formatted with special tokens for the model."
            )

        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )
    return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.num_special_tokens_to_add(pair=False)

Returns the number of added tokens when encoding a sequence with special tokens.

This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop.

PARAMETER	DESCRIPTION
pair	Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
int	int: Number of special tokens added to sequences.

Source code in mindnlp/transformers/tokenization_utils.py

def num_special_tokens_to_add(self, pair: bool = False) -> int:
    """
    Returns the number of added tokens when encoding a sequence with special tokens.

    <Tip>

    This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
    this inside your training loop.

    </Tip>

    Args:
        pair (`bool`, *optional*, defaults to `False`):
            Whether the number of added tokens should be computed in the case of a sequence pair or a single
            sequence.

    Returns:
        `int`: Number of special tokens added to sequences.
    """
    token_ids_0 = []
    token_ids_1 = []
    return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None))

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.prepare_for_tokenization(text, is_split_into_words=False, **kwargs)

Performs any necessary transformations before tokenization.

This method should pop the arguments from kwargs and return the remaining kwargs as well. We test the kwargs at the end of the encoding process to be sure all the arguments have been used.

PARAMETER	DESCRIPTION
text	The text to prepare. TYPE: `str`
is_split_into_words	Whether or not the input is already pre-tokenized (e.g., split into words). If set to True, the tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. This is useful for NER or token classification. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
kwargs	Keyword arguments to use for the tokenization. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

RETURNS	DESCRIPTION
Tuple[str, Dict[str, Any]]	Tuple[str, Dict[str, Any]]: The prepared text and the unused kwargs.

Source code in mindnlp/transformers/tokenization_utils.py

def prepare_for_tokenization(
    self, text: str, is_split_into_words: bool = False, **kwargs
) -> Tuple[str, Dict[str, Any]]:
    """
    Performs any necessary transformations before tokenization.

    This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the
    `kwargs` at the end of the encoding process to be sure all the arguments have been used.

    Args:
        text (`str`):
            The text to prepare.
        is_split_into_words (`bool`, *optional*, defaults to `False`):
            Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the
            tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)
            which it will tokenize. This is useful for NER or token classification.
        kwargs (`Dict[str, Any]`, *optional*):
            Keyword arguments to use for the tokenization.

    Returns:
        `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs.
    """
    return (text, kwargs)

mindnlp.transformers.tokenization_utils.PreTrainedTokenizer.tokenize(text, **kwargs)

Converts a string in a sequence of tokens, using the tokenizer.

Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Takes care of added tokens.

PARAMETER	DESCRIPTION
text	The sequence to be encoded. TYPE: `str`
**kwargs	Passed along to the model-specific prepare_for_tokenization preprocessing method. TYPE: additional keyword arguments DEFAULT: {}

RETURNS	DESCRIPTION
List[str]	List[str]: The list of tokens.

Source code in mindnlp/transformers/tokenization_utils.py

def tokenize(self, text: TextInput, **kwargs) -> List[str]:
    """
    Converts a string in a sequence of tokens, using the tokenizer.

    Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies
    (BPE/SentencePieces/WordPieces). Takes care of added tokens.

    Args:
        text (`str`):
            The sequence to be encoded.
        **kwargs (additional keyword arguments):
            Passed along to the model-specific `prepare_for_tokenization` preprocessing method.

    Returns:
        `List[str]`: The list of tokens.
    """
    split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)

    text, kwargs = self.prepare_for_tokenization(text, **kwargs)

    if kwargs:
        logger.warning(f"Keyword arguments {kwargs} not recognized.")

    if hasattr(self, "do_lower_case") and self.do_lower_case:
        # convert non-special tokens to lowercase. Might be super slow as well?
        escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
        escaped_special_toks += [
            re.escape(s_tok.content)
            for s_tok in (self._added_tokens_decoder.values())
            if not s_tok.special and s_tok.normalized
        ]
        pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
        text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)

    if split_special_tokens:
        no_split_token = []
        tokens = [text]
    else:
        no_split_token = self._added_tokens_encoder.keys()  # don't split on any of the added tokens
        # "This is something<special_token_1>  else"
        tokens = self.tokens_trie.split(text)

    # ["This is something", "<special_token_1>", "  else"]
    for i, token in enumerate(tokens):
        if token in no_split_token:
            tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)
            left = tokens[i - 1] if i > 0 else None
            right = tokens[i + 1] if i < len(tokens) - 1 else None
            if isinstance(tok_extended, AddedToken):
                if tok_extended.rstrip and right:
                    # A bit counter-intuitive but we strip the left of the string
                    # since tok_extended.rstrip means the special token is eating all white spaces on its right
                    tokens[i + 1] = right.lstrip()
                # Strip white spaces on the left
                if tok_extended.lstrip and left:
                    tokens[i - 1] = left.rstrip()  # Opposite here
                if tok_extended.single_word and left and left[-1] != " ":
                    tokens[i - 1] += token
                    tokens[i] = ""
                elif tok_extended.single_word and right and right[0] != " ":
                    tokens[i + 1] = token + tokens[i + 1]
                    tokens[i] = ""
            else:
                raise ValueError(
                    f"{tok_extended} cannot be tokenized because it was not properly added"
                    f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"
                )
    # ["This is something", "<special_token_1>", "else"]
    tokenized_text = []
    for token in tokens:
        # Need to skip eventual empty (fully stripped) tokens
        if not token:
            continue
        if token in no_split_token:
            tokenized_text.append(token)
        else:
            tokenized_text.extend(self._tokenize(token))
    # ["This", " is", " something", "<special_token_1>", "else"]
    return tokenized_text