cpm

mindnlp.transformers.models.cpm.tokenization_cpm

Tokenization classes.

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer

Bases: PreTrainedTokenizer

Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

class CpmTokenizer(PreTrainedTokenizer):
    """Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models."""
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP

    def __init__(
        self,
        vocab_file,
        do_lower_case=False,
        remove_space=True,
        keep_accents=False,
        bos_token="<s>",
        eos_token="</s>",
        unk_token="<unk>",
        sep_token="<sep>",
        pad_token="<pad>",
        cls_token="<cls>",
        mask_token="<mask>",
        additional_special_tokens=["<eop>", "<eod>"],
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ) -> None:
        """
        Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
        [SentencePiece](https://github.com/google/sentencepiece).

        This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
        refer to this superclass for more information regarding those methods.

        Args:
            vocab_file (`str`):
                [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
                contains the vocabulary necessary to instantiate a tokenizer.
            do_lower_case (`bool`, *optional*, defaults to `True`):
                Whether to lowercase the input when tokenizing.
            remove_space (`bool`, *optional*, defaults to `True`):
                Whether to strip the text when tokenizing (removing excess spaces before and after the string).
            keep_accents (`bool`, *optional*, defaults to `False`):
                Whether to keep accents when tokenizing.
            bos_token (`str`, *optional*, defaults to `"<s>"`):
                The beginning of sequence token that was used during pretraining. Can be used a sequence classifier
                token.

                <Tip>

                When building a sequence using special tokens, this is not the token that is used for the beginning of
                sequence. The token used is the `cls_token`.

                </Tip>

            eos_token (`str`, *optional*, defaults to `"</s>"`):
                The end of sequence token.

                <Tip>

                When building a sequence using special tokens, this is not the token that is used for the end of
                sequence. The token used is the `sep_token`.

                </Tip>

            unk_token (`str`, *optional*, defaults to `"<unk>"`):
                The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be
                this token instead.
            sep_token (`str`, *optional*, defaults to `"<sep>"`):
                The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
                for sequence classification or for a text and a question for question answering. It is also used as the
                last token of a sequence built with special tokens.
            pad_token (`str`, *optional*, defaults to `"<pad>"`):
                The token used for padding, for example when batching sequences of different lengths.
            cls_token (`str`, *optional*, defaults to `"<cls>"`):
                The classifier token which is used when doing sequence classification (classification of the whole
                sequence instead of per-token classification). It is the first token of the sequence when built with
                special tokens.
            mask_token (`str`, *optional*, defaults to `"<mask>"`):
                The token used for masking values. This is the token used when training this model with masked language
                modeling. This is the token which the model will try to predict.
            additional_special_tokens (`List[str]`, *optional*, defaults to `["<eop>", "<eod>"]`):
                Additional special tokens used by the tokenizer.

        Attributes:
            sp_model (`SentencePieceProcessor`):
                The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
        """
        # Mask token behave like a normal word, i.e. include the space before it
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.do_lower_case = do_lower_case
        self.remove_space = remove_space
        self.keep_accents = keep_accents
        self.vocab_file = vocab_file

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        try:
            import jieba
        except ModuleNotFoundError as error:
            raise error.__class__(
                "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                "See https://pypi.org/project/jieba/ for installation."
            )
        self.jieba = jieba
        self.translator = str.maketrans(" \n", "\u2582\u2583")

        super().__init__(
            do_lower_case=do_lower_case,
            remove_space=remove_space,
            keep_accents=keep_accents,
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

        self._pad_token_type_id = 3

    @property
    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
    def vocab_size(self):
        """
        Method to retrieve the vocabulary size of the CpmTokenizer instance.

        Args:
            self (CpmTokenizer): The instance of the CpmTokenizer class.
                This parameter is required to access the vocabulary model.

        Returns:
            int: The size of the vocabulary model.
                The return value indicates the number of tokens in the vocabulary model.

        Raises:
            None.
        """
        return len(self.sp_model)

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.get_vocab
    def get_vocab(self):
        """
        Retrieves the vocabulary of the CpmTokenizer instance.

        Args:
            self (CpmTokenizer): The instance of the CpmTokenizer class.

        Returns:
            dict: A dictionary containing the vocabulary of the CpmTokenizer instance.
                The keys are the tokens in the vocabulary, and the values are the corresponding token IDs.
                The vocabulary includes both the default vocabulary and any additional tokens that have been added.

        Raises:
            None.
        """
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.__getstate__
    def __getstate__(self):
        """
        Method '__getstate__' in the class 'CpmTokenizer'.

        Args:
            self: CpmTokenizer
                The instance of the CpmTokenizer class.
                Parameter to access the internal state of the object.

        Returns:
            None:
                Returns the state of the object with 'sp_model' set to None.

        Raises:
            None.
        """
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.__setstate__
    def __setstate__(self, d):
        """
        Method '__setstate__' in the class 'CpmTokenizer' updates the state of the object by restoring its attributes
        from a serialized state dictionary.

        Args:
            self (CpmTokenizer): The instance of the CpmTokenizer class.
            d (dict): The serialized state dictionary containing the attributes to be restored.

        Returns:
            None.

        Raises:
            None.

        This method updates the '__dict__' attribute of the 'self' object with the attributes from the serialized
        state dictionary 'd'. If the 'sp_model_kwargs' attribute doesn't exist in the object, it is initialized as an
        empty dictionary.
        Then, a SentencePieceProcessor object 'sp_model' is created with the keyword arguments provided in
        'self.sp_model_kwargs'.
        Finally, the 'vocab_file' is loaded into the 'sp_model'.
        """
        self.__dict__ = d

        # for backward compatibility
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.preprocess_text
    def preprocess_text(self, inputs):
        """
        This method preprocesses text input based on the specified settings in the CpmTokenizer class.

        Args:
            self (CpmTokenizer): An instance of the CpmTokenizer class.
            inputs (str): The text input to be preprocessed.

        Returns:
            None: This method does not return any value directly. The preprocessed text is stored internally within the method.

        Raises:
            None: This method does not raise any exceptions explicitly.
                However, potential exceptions may arise from the use of external functions within the method such as
                unicodedata.normalize() and  unicodedata.combining().
        """
        if self.remove_space:
            outputs = " ".join(inputs.strip().split())
        else:
            outputs = inputs
        outputs = outputs.replace("``", '"').replace("''", '"')

        if not self.keep_accents:
            outputs = unicodedata.normalize("NFKD", outputs)
            outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
        if self.do_lower_case:
            outputs = outputs.lower()

        return outputs

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer._tokenize
    def _tokenize(self, text: str) -> List[str]:
        """Tokenize a string."""
        text = self.preprocess_text(text)
        pieces = self.sp_model.encode(text, out_type=str)
        new_pieces = []
        for piece in pieces:
            if len(piece) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
                cur_pieces = self.sp_model.EncodeAsPieces(piece[:-1].replace(SENTENCEPIECE_UNDERLINE, ""))
                if piece[0] != SENTENCEPIECE_UNDERLINE and cur_pieces[0][0] == SENTENCEPIECE_UNDERLINE:
                    if len(cur_pieces[0]) == 1:
                        cur_pieces = cur_pieces[1:]
                    else:
                        cur_pieces[0] = cur_pieces[0][1:]
                cur_pieces.append(piece[-1])
                new_pieces.extend(cur_pieces)
            else:
                new_pieces.append(piece)

        return new_pieces

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer._convert_token_to_id
    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.sp_model.PieceToId(token)

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer._convert_id_to_token
    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.sp_model.IdToPiece(index)

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.convert_tokens_to_string
    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (strings for sub-words) in a single string."""
        out_string = "".join(tokens).replace(SENTENCEPIECE_UNDERLINE, " ").strip()
        return out_string

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An XLNet sequence has the following format:

        - single sequence: `X <sep> <cls>`
        - pair of sequences: `A <sep> B <sep> <cls>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return token_ids_0 + sep + cls
        return token_ids_0 + sep + token_ids_1 + sep + cls

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.get_special_tokens_mask
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is not None:
            return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1]
        return ([0] * len(token_ids_0)) + [1, 1]

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.create_token_type_ids_from_sequences
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet
        sequence pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        sep = [self.sep_token_id]
        cls_segment_id = [2]

        if token_ids_1 is None:
            return len(token_ids_0 + sep) * [0] + cls_segment_id
        return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id

    # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.save_vocabulary
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary file to the specified directory with an optional filename prefix.

        Args:
            self: Instance of CpmTokenizer.
            save_directory (str): The directory where the vocabulary file will be saved.
            filename_prefix (Optional[str], optional): An optional prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            IOError: If the save_directory is not a valid directory.
            FileNotFoundError: If the self.vocab_file does not exist.
            Exception: If any other unexpected error occurs during the file operations.
        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

    def _decode(self, *args, **kwargs):
        """
        Method _decode in the class CpmTokenizer.

        Args:
            self: The instance of the CpmTokenizer class.

        Returns:
            None: The method modifies the text content and returns None.

        Raises:
            No specific exceptions are raised within the method.
            However, potential exceptions from the super()._decode() method may be propagated.
        """
        text = super()._decode(*args, **kwargs)
        text = text.replace(" ", "").replace("\u2582", " ").replace("\u2583", "\n")
        return text

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.vocab_size property

Method to retrieve the vocabulary size of the CpmTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the CpmTokenizer class. This parameter is required to access the vocabulary model. TYPE: CpmTokenizer

RETURNS	DESCRIPTION
int	The size of the vocabulary model. The return value indicates the number of tokens in the vocabulary model.

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.__getstate__()

Method 'getstate' in the class 'CpmTokenizer'.

PARAMETER	DESCRIPTION
self	CpmTokenizer The instance of the CpmTokenizer class. Parameter to access the internal state of the object.

RETURNS	DESCRIPTION
None	Returns the state of the object with 'sp_model' set to None.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def __getstate__(self):
    """
    Method '__getstate__' in the class 'CpmTokenizer'.

    Args:
        self: CpmTokenizer
            The instance of the CpmTokenizer class.
            Parameter to access the internal state of the object.

    Returns:
        None:
            Returns the state of the object with 'sp_model' set to None.

    Raises:
        None.
    """
    state = self.__dict__.copy()
    state["sp_model"] = None
    return state

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.__init__(vocab_file, do_lower_case=False, remove_space=True, keep_accents=False, bos_token='<s>', eos_token='</s>', unk_token='<unk>', sep_token='<sep>', pad_token='<pad>', cls_token='<cls>', mask_token='<mask>', additional_special_tokens=['<eop>', '<eod>'], sp_model_kwargs=None, **kwargs)

Construct a CPM tokenizer. Based on Jieba and SentencePiece.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	SentencePiece file (generally has a .spm extension) that contains the vocabulary necessary to instantiate a tokenizer. TYPE: `str`
do_lower_case	Whether to lowercase the input when tokenizing. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: False
remove_space	Whether to strip the text when tokenizing (removing excess spaces before and after the string). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
keep_accents	Whether to keep accents when tokenizing. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the cls_token. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"<sep>"` DEFAULT: '<sep>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"<cls>"` DEFAULT: '<cls>'
mask_token	The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. TYPE: `str`, *optional*, defaults to `"<mask>"` DEFAULT: '<mask>'
additional_special_tokens	Additional special tokens used by the tokenizer. TYPE: `List[str]`, *optional*, defaults to `["<eop>", "<eod>"]` DEFAULT: ['<eop>', '<eod>']

ATTRIBUTE	DESCRIPTION
sp_model	The SentencePiece processor that is used for every conversion (string, tokens and IDs). TYPE: `SentencePieceProcessor`

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def __init__(
    self,
    vocab_file,
    do_lower_case=False,
    remove_space=True,
    keep_accents=False,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    sep_token="<sep>",
    pad_token="<pad>",
    cls_token="<cls>",
    mask_token="<mask>",
    additional_special_tokens=["<eop>", "<eod>"],
    sp_model_kwargs: Optional[Dict[str, Any]] = None,
    **kwargs,
) -> None:
    """
    Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
    [SentencePiece](https://github.com/google/sentencepiece).

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether to lowercase the input when tokenizing.
        remove_space (`bool`, *optional*, defaults to `True`):
            Whether to strip the text when tokenizing (removing excess spaces before and after the string).
        keep_accents (`bool`, *optional*, defaults to `False`):
            Whether to keep accents when tokenizing.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier
            token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of
            sequence. The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be
            this token instead.
        sep_token (`str`, *optional*, defaults to `"<sep>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
            for sequence classification or for a text and a question for question answering. It is also used as the
            last token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"<cls>"`):
            The classifier token which is used when doing sequence classification (classification of the whole
            sequence instead of per-token classification). It is the first token of the sequence when built with
            special tokens.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        additional_special_tokens (`List[str]`, *optional*, defaults to `["<eop>", "<eod>"]`):
            Additional special tokens used by the tokenizer.

    Attributes:
        sp_model (`SentencePieceProcessor`):
            The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
    """
    # Mask token behave like a normal word, i.e. include the space before it
    mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

    self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

    self.do_lower_case = do_lower_case
    self.remove_space = remove_space
    self.keep_accents = keep_accents
    self.vocab_file = vocab_file

    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.Load(vocab_file)

    try:
        import jieba
    except ModuleNotFoundError as error:
        raise error.__class__(
            "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
            "See https://pypi.org/project/jieba/ for installation."
        )
    self.jieba = jieba
    self.translator = str.maketrans(" \n", "\u2582\u2583")

    super().__init__(
        do_lower_case=do_lower_case,
        remove_space=remove_space,
        keep_accents=keep_accents,
        bos_token=bos_token,
        eos_token=eos_token,
        unk_token=unk_token,
        sep_token=sep_token,
        pad_token=pad_token,
        cls_token=cls_token,
        mask_token=mask_token,
        additional_special_tokens=additional_special_tokens,
        sp_model_kwargs=self.sp_model_kwargs,
        **kwargs,
    )

    self._pad_token_type_id = 3

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.__setstate__(d)

Method 'setstate' in the class 'CpmTokenizer' updates the state of the object by restoring its attributes from a serialized state dictionary.

PARAMETER	DESCRIPTION
self	The instance of the CpmTokenizer class. TYPE: CpmTokenizer
d	The serialized state dictionary containing the attributes to be restored. TYPE: dict

RETURNS	DESCRIPTION
	None.

This method updates the 'dict' attribute of the 'self' object with the attributes from the serialized state dictionary 'd'. If the 'sp_model_kwargs' attribute doesn't exist in the object, it is initialized as an empty dictionary. Then, a SentencePieceProcessor object 'sp_model' is created with the keyword arguments provided in 'self.sp_model_kwargs'. Finally, the 'vocab_file' is loaded into the 'sp_model'.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def __setstate__(self, d):
    """
    Method '__setstate__' in the class 'CpmTokenizer' updates the state of the object by restoring its attributes
    from a serialized state dictionary.

    Args:
        self (CpmTokenizer): The instance of the CpmTokenizer class.
        d (dict): The serialized state dictionary containing the attributes to be restored.

    Returns:
        None.

    Raises:
        None.

    This method updates the '__dict__' attribute of the 'self' object with the attributes from the serialized
    state dictionary 'd'. If the 'sp_model_kwargs' attribute doesn't exist in the object, it is initialized as an
    empty dictionary.
    Then, a SentencePieceProcessor object 'sp_model' is created with the keyword arguments provided in
    'self.sp_model_kwargs'.
    Finally, the 'vocab_file' is loaded into the 'sp_model'.
    """
    self.__dict__ = d

    # for backward compatibility
    if not hasattr(self, "sp_model_kwargs"):
        self.sp_model_kwargs = {}

    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.Load(self.vocab_file)

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. An XLNet sequence has the following format:

• single sequence: X <sep> <cls>
• pair of sequences: A <sep> B <sep> <cls>

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. An XLNet sequence has the following format:

    - single sequence: `X <sep> <cls>`
    - pair of sequences: `A <sep> B <sep> <cls>`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return token_ids_0 + sep + cls
    return token_ids_0 + sep + token_ids_1 + sep + cls

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (strings for sub-words) in a single string.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (strings for sub-words) in a single string."""
    out_string = "".join(tokens).replace(SENTENCEPIECE_UNDERLINE, " ").strip()
    return out_string

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

If token_ids_1 is None, this method only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet
    sequence pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    sep = [self.sep_token_id]
    cls_segment_id = [2]

    if token_ids_1 is None:
        return len(token_ids_0 + sep) * [0] + cls_segment_id
    return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    if token_ids_1 is not None:
        return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1]
    return ([0] * len(token_ids_0)) + [1, 1]

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.get_vocab()

Retrieves the vocabulary of the CpmTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the CpmTokenizer class. TYPE: CpmTokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the vocabulary of the CpmTokenizer instance. The keys are the tokens in the vocabulary, and the values are the corresponding token IDs. The vocabulary includes both the default vocabulary and any additional tokens that have been added.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def get_vocab(self):
    """
    Retrieves the vocabulary of the CpmTokenizer instance.

    Args:
        self (CpmTokenizer): The instance of the CpmTokenizer class.

    Returns:
        dict: A dictionary containing the vocabulary of the CpmTokenizer instance.
            The keys are the tokens in the vocabulary, and the values are the corresponding token IDs.
            The vocabulary includes both the default vocabulary and any additional tokens that have been added.

    Raises:
        None.
    """
    vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
    vocab.update(self.added_tokens_encoder)
    return vocab

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.preprocess_text(inputs)

This method preprocesses text input based on the specified settings in the CpmTokenizer class.

PARAMETER	DESCRIPTION
self	An instance of the CpmTokenizer class. TYPE: CpmTokenizer
inputs	The text input to be preprocessed. TYPE: str

RETURNS	DESCRIPTION
None	This method does not return any value directly. The preprocessed text is stored internally within the method.

RAISES	DESCRIPTION
None	This method does not raise any exceptions explicitly. However, potential exceptions may arise from the use of external functions within the method such as unicodedata.normalize() and unicodedata.combining().

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def preprocess_text(self, inputs):
    """
    This method preprocesses text input based on the specified settings in the CpmTokenizer class.

    Args:
        self (CpmTokenizer): An instance of the CpmTokenizer class.
        inputs (str): The text input to be preprocessed.

    Returns:
        None: This method does not return any value directly. The preprocessed text is stored internally within the method.

    Raises:
        None: This method does not raise any exceptions explicitly.
            However, potential exceptions may arise from the use of external functions within the method such as
            unicodedata.normalize() and  unicodedata.combining().
    """
    if self.remove_space:
        outputs = " ".join(inputs.strip().split())
    else:
        outputs = inputs
    outputs = outputs.replace("``", '"').replace("''", '"')

    if not self.keep_accents:
        outputs = unicodedata.normalize("NFKD", outputs)
        outputs = "".join([c for c in outputs if not unicodedata.combining(c)])
    if self.do_lower_case:
        outputs = outputs.lower()

    return outputs

mindnlp.transformers.models.cpm.tokenization_cpm.CpmTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary file to the specified directory with an optional filename prefix.

PARAMETER	DESCRIPTION
self	Instance of CpmTokenizer.
save_directory	The directory where the vocabulary file will be saved. TYPE: str
filename_prefix	An optional prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
IOError	If the save_directory is not a valid directory.
FileNotFoundError	If the self.vocab_file does not exist.
Exception	If any other unexpected error occurs during the file operations.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary file to the specified directory with an optional filename prefix.

    Args:
        self: Instance of CpmTokenizer.
        save_directory (str): The directory where the vocabulary file will be saved.
        filename_prefix (Optional[str], optional): An optional prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        IOError: If the save_directory is not a valid directory.
        FileNotFoundError: If the self.vocab_file does not exist.
        Exception: If any other unexpected error occurs during the file operations.
    """
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file, out_vocab_file)
    elif not os.path.isfile(self.vocab_file):
        with open(out_vocab_file, "wb") as fi:
            content_spiece_model = self.sp_model.serialized_model_proto()
            fi.write(content_spiece_model)

    return (out_vocab_file,)

mindnlp.transformers.models.cpm.tokenization_cpm_fast

Tokenization classes.

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast

Bases: PreTrainedTokenizerFast

Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm_fast.py

class CpmTokenizerFast(PreTrainedTokenizerFast):
    """Runs pre-tokenization with Jieba segmentation tool. It is used in CPM models."""
    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        do_lower_case=False,
        remove_space=True,
        keep_accents=False,
        bos_token="<s>",
        eos_token="</s>",
        unk_token="<unk>",
        sep_token="<sep>",
        pad_token="<pad>",
        cls_token="<cls>",
        mask_token="<mask>",
        additional_special_tokens=["<eop>", "<eod>"],
        **kwargs,
    ):
        """
        Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
        [SentencePiece](https://github.com/google/sentencepiece).

        This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
        refer to this superclass for more information regarding those methods.

        Args:
            vocab_file (`str`):
                [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
                contains the vocabulary necessary to instantiate a tokenizer.
            do_lower_case (`bool`, *optional*, defaults to `True`):
                Whether to lowercase the input when tokenizing.
            remove_space (`bool`, *optional*, defaults to `True`):
                Whether to strip the text when tokenizing (removing excess spaces before and after the string).
            keep_accents (`bool`, *optional*, defaults to `False`):
                Whether to keep accents when tokenizing.
            bos_token (`str`, *optional*, defaults to `"<s>"`):
                The beginning of sequence token that was used during pretraining. Can be used a sequence classifier
                token.

                <Tip>

                When building a sequence using special tokens, this is not the token that is used for the beginning of
                sequence. The token used is the `cls_token`.

                </Tip>

            eos_token (`str`, *optional*, defaults to `"</s>"`):
                The end of sequence token.

                <Tip>

                When building a sequence using special tokens, this is not the token that is used for the end of
                sequence. The token used is the `sep_token`.

                </Tip>

            unk_token (`str`, *optional*, defaults to `"<unk>"`):
                The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be
                this token instead.
            sep_token (`str`, *optional*, defaults to `"<sep>"`):
                The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
                for sequence classification or for a text and a question for question answering. It is also used as the
                last token of a sequence built with special tokens.
            pad_token (`str`, *optional*, defaults to `"<pad>"`):
                The token used for padding, for example when batching sequences of different lengths.
            cls_token (`str`, *optional*, defaults to `"<cls>"`):
                The classifier token which is used when doing sequence classification (classification of the whole
                sequence instead of per-token classification). It is the first token of the sequence when built with
                special tokens.
            mask_token (`str`, *optional*, defaults to `"<mask>"`):
                The token used for masking values. This is the token used when training this model with masked language
                modeling. This is the token which the model will try to predict.
            additional_special_tokens (`List[str]`, *optional*, defaults to `["<eop>", "<eod>"]`):
                Additional special tokens used by the tokenizer.

        Attributes:
            sp_model (`SentencePieceProcessor`):
                The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
        """
        # Mask token behave like a normal word, i.e. include the space before it
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        super().__init__(
            vocab_file=vocab_file,
            tokenizer_file=tokenizer_file,
            do_lower_case=do_lower_case,
            remove_space=remove_space,
            keep_accents=keep_accents,
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self._pad_token_type_id = 3
        self.do_lower_case = do_lower_case
        self.remove_space = remove_space
        self.keep_accents = keep_accents
        self.vocab_file = vocab_file

        try:
            import jieba
        except ModuleNotFoundError as error:
            raise error.__class__(
                "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
                "See https://pypi.org/project/jieba/ for installation."
            )
        self.jieba = jieba
        self.translator = str.maketrans(" \n", "\u2582\u2583")

    @property
    def can_save_slow_tokenizer(self) -> bool:
        """
        Method: can_save_slow_tokenizer

        Description:
        This method checks if the slow tokenizer can be saved by verifying the existence of the vocabulary file.

        Args:
            self: The instance of the CpmTokenizerFast class.

        Returns:
            bool: Returns a boolean value indicating whether the slow tokenizer can be saved.
                Returns True if the vocabulary file exists, otherwise False.

        Raises:
            This method does not raise any exceptions.
        """
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    # Copied from transformers.models.xlnet.tokenization_xlnet_fast.XLNetTokenizerFast.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An XLNet sequence has the following format:

        - single sequence: `X <sep> <cls>`
        - pair of sequences: `A <sep> B <sep> <cls>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return token_ids_0 + sep + cls
        return token_ids_0 + sep + token_ids_1 + sep + cls

    # Copied from transformers.models.xlnet.tokenization_xlnet_fast.XLNetTokenizerFast.create_token_type_ids_from_sequences
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet
        sequence pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        sep = [self.sep_token_id]
        cls_segment_id = [2]

        if token_ids_1 is None:
            return len(token_ids_0 + sep) * [0] + cls_segment_id
        return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id

    # Copied from transformers.models.xlnet.tokenization_xlnet_fast.XLNetTokenizerFast.save_vocabulary
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Saves the vocabulary of a fast tokenizer to a specified directory.

        Args:
            self (CpmTokenizerFast): The instance of the fast tokenizer.
            save_directory (str): The directory where the vocabulary will be saved.
            filename_prefix (Optional[str], optional): The prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            ValueError: Raised if the fast tokenizer does not have the necessary information to save the vocabulary
                for a slow tokenizer.
            OSError: Raised if the save_directory is not a valid directory.

        Note:
            The method assumes that the fast tokenizer has the required information to save the vocabulary for
            a slow tokenizer. If this is not the case, a ValueError is raised.

        Example:
            ```python
            >>> tokenizer = CpmTokenizerFast()
            >>> tokenizer.save_vocabulary('path/to/save', 'vocab')
            ```
        """
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)

    def _batch_encode_plus(self, batch_text_or_text_pairs, *args, **kwargs):
        """
        Performs batch encoding of text or text pairs using the CpmTokenizerFast class.

        Args:
            self (CpmTokenizerFast): An instance of the CpmTokenizerFast class.
            batch_text_or_text_pairs (list or tuple): A list or tuple containing either text or text pairs to encode.
                If a list of text is provided, each text item will be encoded individually.
                If a list of text pairs is provided, each pair will be encoded as a single unit.
                The text can be in any language, but it must be preprocessed and tokenized before passing it to this method.

        Returns:
            None

        Raises:
            None

        Note:
            - This method internally uses the _batch_encode_plus method of the superclass.
            - The text is preprocessed using the jieba library to tokenize Chinese text.
            - The translator attribute of the CpmTokenizerFast instance is used to remove certain characters from the text.
            - Other arguments and keyword arguments passed to this method will be forwarded to the superclass method.
        """
        batch_text_or_text_pairs = [
            " ".join([x.translate(self.translator) for x in self.jieba.cut(text, cut_all=False)])
            for text in batch_text_or_text_pairs
        ]
        return super()._batch_encode_plus(batch_text_or_text_pairs, *args, **kwargs)

    def _decode(self, *args, **kwargs):
        """
        Decodes the text representation of a CpmTokenizerFast object.

        Args:
            self: An instance of the CpmTokenizerFast class.

        Returns:
            None: This method modifies the text representation of the CpmTokenizerFast object in-place.

        Raises:
            None.
        """
        text = super()._decode(*args, **kwargs)
        text = text.replace(" ", "").replace("\u2582", " ").replace("\u2583", "\n")
        return text

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast.can_save_slow_tokenizer: bool property

Description: This method checks if the slow tokenizer can be saved by verifying the existence of the vocabulary file.

PARAMETER	DESCRIPTION
self	The instance of the CpmTokenizerFast class.

RETURNS	DESCRIPTION
bool	Returns a boolean value indicating whether the slow tokenizer can be saved. Returns True if the vocabulary file exists, otherwise False. TYPE: bool

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast.__init__(vocab_file=None, tokenizer_file=None, do_lower_case=False, remove_space=True, keep_accents=False, bos_token='<s>', eos_token='</s>', unk_token='<unk>', sep_token='<sep>', pad_token='<pad>', cls_token='<cls>', mask_token='<mask>', additional_special_tokens=['<eop>', '<eod>'], **kwargs)

Construct a CPM tokenizer. Based on Jieba and SentencePiece.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	SentencePiece file (generally has a .spm extension) that contains the vocabulary necessary to instantiate a tokenizer. TYPE: `str` DEFAULT: None
do_lower_case	Whether to lowercase the input when tokenizing. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: False
remove_space	Whether to strip the text when tokenizing (removing excess spaces before and after the string). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
keep_accents	Whether to keep accents when tokenizing. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. When building a sequence using special tokens, this is not the token that is used for the beginning of sequence. The token used is the cls_token. TYPE: `str`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. When building a sequence using special tokens, this is not the token that is used for the end of sequence. The token used is the sep_token. TYPE: `str`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
sep_token	The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for sequence classification or for a text and a question for question answering. It is also used as the last token of a sequence built with special tokens. TYPE: `str`, *optional*, defaults to `"<sep>"` DEFAULT: '<sep>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional*, defaults to `"<pad>"` DEFAULT: '<pad>'
cls_token	The classifier token which is used when doing sequence classification (classification of the whole sequence instead of per-token classification). It is the first token of the sequence when built with special tokens. TYPE: `str`, *optional*, defaults to `"<cls>"` DEFAULT: '<cls>'
mask_token	The token used for masking values. This is the token used when training this model with masked language modeling. This is the token which the model will try to predict. TYPE: `str`, *optional*, defaults to `"<mask>"` DEFAULT: '<mask>'
additional_special_tokens	Additional special tokens used by the tokenizer. TYPE: `List[str]`, *optional*, defaults to `["<eop>", "<eod>"]` DEFAULT: ['<eop>', '<eod>']

ATTRIBUTE	DESCRIPTION
sp_model	The SentencePiece processor that is used for every conversion (string, tokens and IDs). TYPE: `SentencePieceProcessor`

Source code in mindnlp/transformers/models/cpm/tokenization_cpm_fast.py

def __init__(
    self,
    vocab_file=None,
    tokenizer_file=None,
    do_lower_case=False,
    remove_space=True,
    keep_accents=False,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    sep_token="<sep>",
    pad_token="<pad>",
    cls_token="<cls>",
    mask_token="<mask>",
    additional_special_tokens=["<eop>", "<eod>"],
    **kwargs,
):
    """
    Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
    [SentencePiece](https://github.com/google/sentencepiece).

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether to lowercase the input when tokenizing.
        remove_space (`bool`, *optional*, defaults to `True`):
            Whether to strip the text when tokenizing (removing excess spaces before and after the string).
        keep_accents (`bool`, *optional*, defaults to `False`):
            Whether to keep accents when tokenizing.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier
            token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the beginning of
            sequence. The token used is the `cls_token`.

            </Tip>

        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of
            sequence. The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be
            this token instead.
        sep_token (`str`, *optional*, defaults to `"<sep>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences
            for sequence classification or for a text and a question for question answering. It is also used as the
            last token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"<cls>"`):
            The classifier token which is used when doing sequence classification (classification of the whole
            sequence instead of per-token classification). It is the first token of the sequence when built with
            special tokens.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        additional_special_tokens (`List[str]`, *optional*, defaults to `["<eop>", "<eod>"]`):
            Additional special tokens used by the tokenizer.

    Attributes:
        sp_model (`SentencePieceProcessor`):
            The *SentencePiece* processor that is used for every conversion (string, tokens and IDs).
    """
    # Mask token behave like a normal word, i.e. include the space before it
    mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

    super().__init__(
        vocab_file=vocab_file,
        tokenizer_file=tokenizer_file,
        do_lower_case=do_lower_case,
        remove_space=remove_space,
        keep_accents=keep_accents,
        bos_token=bos_token,
        eos_token=eos_token,
        unk_token=unk_token,
        sep_token=sep_token,
        pad_token=pad_token,
        cls_token=cls_token,
        mask_token=mask_token,
        additional_special_tokens=additional_special_tokens,
        **kwargs,
    )

    self._pad_token_type_id = 3
    self.do_lower_case = do_lower_case
    self.remove_space = remove_space
    self.keep_accents = keep_accents
    self.vocab_file = vocab_file

    try:
        import jieba
    except ModuleNotFoundError as error:
        raise error.__class__(
            "You need to install jieba to use CpmTokenizer or CpmTokenizerFast. "
            "See https://pypi.org/project/jieba/ for installation."
        )
    self.jieba = jieba
    self.translator = str.maketrans(" \n", "\u2582\u2583")

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. An XLNet sequence has the following format:

• single sequence: X <sep> <cls>
• pair of sequences: A <sep> B <sep> <cls>

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/cpm/tokenization_cpm_fast.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. An XLNet sequence has the following format:

    - single sequence: `X <sep> <cls>`
    - pair of sequences: `A <sep> B <sep> <cls>`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return token_ids_0 + sep + cls
    return token_ids_0 + sep + token_ids_1 + sep + cls

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

If token_ids_1 is None, this method only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/cpm/tokenization_cpm_fast.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLNet
    sequence pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    sep = [self.sep_token_id]
    cls_segment_id = [2]

    if token_ids_1 is None:
        return len(token_ids_0 + sep) * [0] + cls_segment_id
    return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id

mindnlp.transformers.models.cpm.tokenization_cpm_fast.CpmTokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Saves the vocabulary of a fast tokenizer to a specified directory.

PARAMETER	DESCRIPTION
self	The instance of the fast tokenizer. TYPE: CpmTokenizerFast
save_directory	The directory where the vocabulary will be saved. TYPE: str
filename_prefix	The prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
ValueError	Raised if the fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.
OSError	Raised if the save_directory is not a valid directory.

Note

The method assumes that the fast tokenizer has the required information to save the vocabulary for a slow tokenizer. If this is not the case, a ValueError is raised.

Example

>>> tokenizer = CpmTokenizerFast()
>>> tokenizer.save_vocabulary('path/to/save', 'vocab')

Source code in mindnlp/transformers/models/cpm/tokenization_cpm_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Saves the vocabulary of a fast tokenizer to a specified directory.

    Args:
        self (CpmTokenizerFast): The instance of the fast tokenizer.
        save_directory (str): The directory where the vocabulary will be saved.
        filename_prefix (Optional[str], optional): The prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        ValueError: Raised if the fast tokenizer does not have the necessary information to save the vocabulary
            for a slow tokenizer.
        OSError: Raised if the save_directory is not a valid directory.

    Note:
        The method assumes that the fast tokenizer has the required information to save the vocabulary for
        a slow tokenizer. If this is not the case, a ValueError is raised.

    Example:
        ```python
        >>> tokenizer = CpmTokenizerFast()
        >>> tokenizer.save_vocabulary('path/to/save', 'vocab')
        ```
    """
    if not self.can_save_slow_tokenizer:
        raise ValueError(
            "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
            "tokenizer."
        )

    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    out_vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )

    if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
        copyfile(self.vocab_file, out_vocab_file)

    return (out_vocab_file,)