bert_japanese

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer

Bases: PreTrainedTokenizer

Construct a BERT tokenizer for Japanese text.

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to: this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	Path to a one-wordpiece-per-line vocabulary file. TYPE: `str`
spm_file	Path to SentencePiece file (generally has a .spm or .model extension) that contains the vocabulary. TYPE: `str`, *optional* DEFAULT: None
do_lower_case	Whether to lower case the input. Only has an effect when do_basic_tokenize=True. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: False
do_word_tokenize	Whether to do word tokenization. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
do_subword_tokenize	Whether to do subword tokenization. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
word_tokenizer_type	Type of word tokenizer. Choose from ["basic", "mecab", "sudachi", "jumanpp"]. TYPE: `str`, *optional*, defaults to `"basic"` DEFAULT: 'basic'
subword_tokenizer_type	Type of subword tokenizer. Choose from ["wordpiece", "character", "sentencepiece",]. TYPE: `str`, *optional*, defaults to `"wordpiece"` DEFAULT: 'wordpiece'
mecab_kwargs	Dictionary passed to the MecabTokenizer forwardor. TYPE: `dict`, *optional* DEFAULT: None
sudachi_kwargs	Dictionary passed to the SudachiTokenizer forwardor. TYPE: `dict`, *optional* DEFAULT: None
jumanpp_kwargs	Dictionary passed to the JumanppTokenizer forwardor. TYPE: `dict`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

class BertJapaneseTokenizer(PreTrainedTokenizer):
    r"""
    Construct a BERT tokenizer for Japanese text.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer
    to: this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to a one-wordpiece-per-line vocabulary file.
        spm_file (`str`, *optional*):
            Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm or .model
            extension) that contains the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether to lower case the input. Only has an effect when do_basic_tokenize=True.
        do_word_tokenize (`bool`, *optional*, defaults to `True`):
            Whether to do word tokenization.
        do_subword_tokenize (`bool`, *optional*, defaults to `True`):
            Whether to do subword tokenization.
        word_tokenizer_type (`str`, *optional*, defaults to `"basic"`):
            Type of word tokenizer. Choose from ["basic", "mecab", "sudachi", "jumanpp"].
        subword_tokenizer_type (`str`, *optional*, defaults to `"wordpiece"`):
            Type of subword tokenizer. Choose from ["wordpiece", "character", "sentencepiece",].
        mecab_kwargs (`dict`, *optional*):
            Dictionary passed to the `MecabTokenizer` forwardor.
        sudachi_kwargs (`dict`, *optional*):
            Dictionary passed to the `SudachiTokenizer` forwardor.
        jumanpp_kwargs (`dict`, *optional*):
            Dictionary passed to the `JumanppTokenizer` forwardor.
    """
    vocab_files_names = VOCAB_FILES_NAMES

    def __init__(
        self,
        vocab_file,
        spm_file=None,
        do_lower_case=False,
        do_word_tokenize=True,
        do_subword_tokenize=True,
        word_tokenizer_type="basic",
        subword_tokenizer_type="wordpiece",
        never_split=None,
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        mecab_kwargs=None,
        sudachi_kwargs=None,
        jumanpp_kwargs=None,
        **kwargs,
    ):
        """
        Initializes a new instance of the BertJapaneseTokenizer class.

        Args:
            self (object): The instance of the class.
            vocab_file (str): The path to the vocabulary file. If not using a 'sentencepiece' subword tokenizer, this file is required.
            spm_file (str, optional): The path to the SentencePiece model file. Defaults to None.
            do_lower_case (bool): A flag to indicate whether the tokenizer should convert all characters to lowercase. Defaults to False.
            do_word_tokenize (bool): A flag to indicate whether word tokenization should be performed. Defaults to True.
            do_subword_tokenize (bool): A flag to indicate whether subword tokenization should be performed. Defaults to True.
            word_tokenizer_type (str): The type of word tokenizer to use. Must be one of 'basic', 'mecab', 'sudachi', or 'jumanpp'.
            subword_tokenizer_type (str): The type of subword tokenizer to use. Must be one of 'wordpiece', 'character', or 'sentencepiece'.
            never_split (list): A list of tokens that should not be split during tokenization. Defaults to None.
            unk_token (str): The token to represent unknown words. Defaults to '[UNK]'.
            sep_token (str): The separator token. Defaults to '[SEP]'.
            pad_token (str): The padding token. Defaults to '[PAD]'.
            cls_token (str): The classification token. Defaults to '[CLS]'.
            mask_token (str): The mask token. Defaults to '[MASK]'.
            mecab_kwargs (dict): Additional keyword arguments for the Mecab word tokenizer. Defaults to None.
            sudachi_kwargs (dict): Additional keyword arguments for the Sudachi word tokenizer. Defaults to None.
            jumanpp_kwargs (dict): Additional keyword arguments for the Jumanpp word tokenizer. Defaults to None.

        Returns:
            None.

        Raises:
            ValueError: If the specified vocabulary or SentencePiece model file cannot be found, or if an invalid tokenizer type is specified.
        """
        if subword_tokenizer_type == "sentencepiece":
            if not os.path.isfile(spm_file):
                raise ValueError(
                    f"Can't find a vocabulary file at path '{spm_file}'. To load the vocabulary from a Google"
                    " pretrained model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
                )
            self.spm_file = spm_file
        else:
            if not os.path.isfile(vocab_file):
                raise ValueError(
                    f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google"
                    " pretrained model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
                )
            self.vocab = load_vocab(vocab_file)
            self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])

        self.do_word_tokenize = do_word_tokenize
        self.word_tokenizer_type = word_tokenizer_type
        self.lower_case = do_lower_case
        self.never_split = never_split
        self.mecab_kwargs = copy.deepcopy(mecab_kwargs)
        self.sudachi_kwargs = copy.deepcopy(sudachi_kwargs)
        self.jumanpp_kwargs = copy.deepcopy(jumanpp_kwargs)
        if do_word_tokenize:
            if word_tokenizer_type == "basic":
                self.word_tokenizer = BasicTokenizer(
                    do_lower_case=do_lower_case, never_split=never_split, tokenize_chinese_chars=False
                )
            elif word_tokenizer_type == "mecab":
                self.word_tokenizer = MecabTokenizer(
                    do_lower_case=do_lower_case, never_split=never_split, **(mecab_kwargs or {})
                )
            elif word_tokenizer_type == "sudachi":
                self.word_tokenizer = SudachiTokenizer(
                    do_lower_case=do_lower_case, never_split=never_split, **(sudachi_kwargs or {})
                )
            elif word_tokenizer_type == "jumanpp":
                self.word_tokenizer = JumanppTokenizer(
                    do_lower_case=do_lower_case, never_split=never_split, **(jumanpp_kwargs or {})
                )
            else:
                raise ValueError(f"Invalid word_tokenizer_type '{word_tokenizer_type}' is specified.")

        self.do_subword_tokenize = do_subword_tokenize
        self.subword_tokenizer_type = subword_tokenizer_type
        if do_subword_tokenize:
            if subword_tokenizer_type == "wordpiece":
                self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))
            elif subword_tokenizer_type == "character":
                self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, unk_token=str(unk_token))
            elif subword_tokenizer_type == "sentencepiece":
                self.subword_tokenizer = SentencepieceTokenizer(vocab=self.spm_file, unk_token=str(unk_token))
            else:
                raise ValueError(f"Invalid subword_tokenizer_type '{subword_tokenizer_type}' is specified.")
        super().__init__(
            spm_file=spm_file,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            do_lower_case=do_lower_case,
            do_word_tokenize=do_word_tokenize,
            do_subword_tokenize=do_subword_tokenize,
            word_tokenizer_type=word_tokenizer_type,
            subword_tokenizer_type=subword_tokenizer_type,
            never_split=never_split,
            mecab_kwargs=mecab_kwargs,
            sudachi_kwargs=sudachi_kwargs,
            jumanpp_kwargs=jumanpp_kwargs,
            **kwargs,
        )

    @property
    def do_lower_case(self):
        """
        Method: do_lower_case

        Description:
            This method returns the lower case value of the input.

        Args:
            self: Represents the instance of the class BertJapaneseTokenizer. 
                It is used to access the attributes and methods of the class.

        Returns:
            None: This method does not return any value, 
                rather it directly accesses and returns the lower case value of the input.

        Raises:
            None.
        """
        return self.lower_case

    def __getstate__(self):
        """
        This method '__getstate__' is defined within the class 'BertJapaneseTokenizer' 
        and is used to retrieve the internal state of the object for pickling purposes.

        Args:
            self: This parameter represents the instance of the 'BertJapaneseTokenizer' class itself. 
                It is required to access the internal attributes of the object.

        Returns:
            a dictionary representing the current state of the object:
                If the 'word_tokenizer_type' attribute of the object is one of 
                ['mecab', 'sudachi', 'jumanpp'], the 'word_tokenizer'

                attribute is removed from the state dictionary before returning it.

        Raises:
            None.
        """
        state = dict(self.__dict__)
        if self.word_tokenizer_type in ["mecab", "sudachi", "jumanpp"]:
            del state["word_tokenizer"]
        return state

    def __setstate__(self, state):
        """
        Args:
            self (BertJapaneseTokenizer): The instance of the BertJapaneseTokenizer class.
            state (dict): The state dictionary containing the attributes to be restored.

        Returns:
            None.

        Raises:
            None
        """
        self.__dict__ = state
        if self.word_tokenizer_type == "mecab":
            self.word_tokenizer = MecabTokenizer(
                do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.mecab_kwargs or {})
            )
        elif self.word_tokenizer_type == "sudachi":
            self.word_tokenizer = SudachiTokenizer(
                do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.sudachi_kwargs or {})
            )
        elif self.word_tokenizer_type == "jumanpp":
            self.word_tokenizer = JumanppTokenizer(
                do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.jumanpp_kwargs or {})
            )

    def _tokenize(self, text):
        """
        Tokenizes the given text using word and subword tokenization.

        Args:
            self (BertJapaneseTokenizer): The instance of the BertJapaneseTokenizer class.
            text (str): The text to be tokenized.

        Returns:
            list: The list of tokens after word and subword tokenization.

        Raises:
            None.

        This method tokenizes the input text using word and subword tokenization techniques.
        If the 'do_word_tokenize' flag is set to True, the text is first tokenized into words using the 'word_tokenizer'
        with the 'never_split' option set to 'all_special_tokens'. If the flag is set to False, the text is treated
        as a single token.

        If the 'do_subword_tokenize' flag is set to True, each word token is further split into subword tokens using
        the 'subword_tokenizer'. The resulting subword tokens are returned as the final list of tokens. If the flag
        is set to False, the word tokens are returned as is.

        Note: The 'do_word_tokenize' and 'do_subword_tokenize' flags are set during the initialization of the
        BertJapaneseTokenizer class.

        Example:
            ```python
            >>> tokenizer = BertJapaneseTokenizer()
            >>> text = "こんにちは、世界！"
            >>> tokens = tokenizer._tokenize(text)
            >>> print(tokens)
            >>> # Output: ['こんにちは', '、', '世界', '！']
            ```
        """
        if self.do_word_tokenize:
            tokens = self.word_tokenizer.tokenize(text, never_split=self.all_special_tokens)
        else:
            tokens = [text]

        if self.do_subword_tokenize:
            split_tokens = [sub_token for token in tokens for sub_token in self.subword_tokenizer.tokenize(token)]
        else:
            split_tokens = tokens

        return split_tokens

    @property
    def vocab_size(self):
        """
        This method 'vocab_size' in the class 'BertJapaneseTokenizer' retrieves the vocabulary size
        based on the tokenizer type.

        Args:
            self (object): The instance of the BertJapaneseTokenizer class.

        Returns:
            None: This method does not return any value directly.
                The vocabulary size is accessed through the property 'vocab_size'.

        Raises:
            None
        """
        if self.subword_tokenizer_type == "sentencepiece":
            return len(self.subword_tokenizer.sp_model)
        return len(self.vocab)

    def get_vocab(self):
        """
        This method 'get_vocab' in the class 'BertJapaneseTokenizer' retrieves the vocabulary used by the tokenizer.

        Args:
            self: The instance of the BertJapaneseTokenizer class. It is a required parameter for instance method access.

        Returns:
            a dictionary representing the vocabulary:

                - If the subword_tokenizer_type is 'sentencepiece', the vocabulary is forwarded by mapping token IDs
                to their corresponding tokens for the range of 0 to vocab_size.
                Any added tokens are then added to this vocabulary.
                - If the subword_tokenizer_type is not 'sentencepiece', the vocabulary is a combination of
                the existing vocabulary and the added_tokens_encoder.

        Raises:
            No specific exceptions are documented to be raised by this method.
        """
        if self.subword_tokenizer_type == "sentencepiece":
            vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
            vocab.update(self.added_tokens_encoder)
            return vocab
        return dict(self.vocab, **self.added_tokens_encoder)

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        if self.subword_tokenizer_type == "sentencepiece":
            return self.subword_tokenizer.sp_model.PieceToId(token)
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        if self.subword_tokenizer_type == "sentencepiece":
            return self.subword_tokenizer.sp_model.IdToPiece(index)
        return self.ids_to_tokens.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        if self.subword_tokenizer_type == "sentencepiece":
            return self.subword_tokenizer.sp_model.decode(tokens)
        out_string = " ".join(tokens).replace(" ##", "").strip()
        return out_string

    # Copied from transformers.models.bert.tokenization_bert.BertTokenizer.build_inputs_with_special_tokens
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BERT sequence has the following format:

        - single sequence: `[CLS] X [SEP]`
        - pair of sequences: `[CLS] A [SEP] B [SEP]`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        """
        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    # Copied from transformers.models.bert.tokenization_bert.BertTokenizer.get_special_tokens_mask
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is not None:
            return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1]

    # Copied from transformers.models.bert.tokenization_bert.BertTokenizer.create_token_type_ids_from_sequences
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
        pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary to a file in the specified directory.

        Args:
            self: Instance of the BertJapaneseTokenizer class.
            save_directory (str): The directory where the vocabulary file will be saved.
            filename_prefix (Optional[str]): Optional prefix to be added to the filename. Defaults to None.

        Returns:
            Tuple[str]: A tuple containing the path to the saved vocabulary file.

        Raises:
            FileNotFoundError: If the specified save_directory does not exist.
            ValueError: If the subword_tokenizer_type is not supported.
            IOError: If there is an issue writing the vocabulary file to the specified location.
        """
        if os.path.isdir(save_directory):
            if self.subword_tokenizer_type == "sentencepiece":
                vocab_file = os.path.join(
                    save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["spm_file"]
                )
            else:
                vocab_file = os.path.join(
                    save_directory,
                    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"],
                )
        else:
            vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory

        if self.subword_tokenizer_type == "sentencepiece":
            with open(vocab_file, "wb") as writer:
                content_spiece_model = self.subword_tokenizer.sp_model.serialized_model_proto()
                writer.write(content_spiece_model)
        else:
            with open(vocab_file, "w", encoding="utf-8") as writer:
                index = 0
                for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
                    if index != token_index:
                        logger.warning(
                            f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                            " Please check that the vocabulary is not corrupted!"
                        )
                        index = token_index
                    writer.write(token + "\n")
                    index += 1
        return (vocab_file,)

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.do_lower_case property

Description

This method returns the lower case value of the input.

PARAMETER	DESCRIPTION
self	Represents the instance of the class BertJapaneseTokenizer. It is used to access the attributes and methods of the class.

RETURNS	DESCRIPTION
None	This method does not return any value, rather it directly accesses and returns the lower case value of the input.

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.vocab_size property

This method 'vocab_size' in the class 'BertJapaneseTokenizer' retrieves the vocabulary size based on the tokenizer type.

PARAMETER	DESCRIPTION
self	The instance of the BertJapaneseTokenizer class. TYPE: object

RETURNS	DESCRIPTION
None	This method does not return any value directly. The vocabulary size is accessed through the property 'vocab_size'.

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.__getstate__()

This method 'getstate' is defined within the class 'BertJapaneseTokenizer' and is used to retrieve the internal state of the object for pickling purposes.

PARAMETER	DESCRIPTION
self	This parameter represents the instance of the 'BertJapaneseTokenizer' class itself. It is required to access the internal attributes of the object.

RETURNS	DESCRIPTION
	a dictionary representing the current state of the object: If the 'word_tokenizer_type' attribute of the object is one of ['mecab', 'sudachi', 'jumanpp'], the 'word_tokenizer' attribute is removed from the state dictionary before returning it.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def __getstate__(self):
    """
    This method '__getstate__' is defined within the class 'BertJapaneseTokenizer' 
    and is used to retrieve the internal state of the object for pickling purposes.

    Args:
        self: This parameter represents the instance of the 'BertJapaneseTokenizer' class itself. 
            It is required to access the internal attributes of the object.

    Returns:
        a dictionary representing the current state of the object:
            If the 'word_tokenizer_type' attribute of the object is one of 
            ['mecab', 'sudachi', 'jumanpp'], the 'word_tokenizer'

            attribute is removed from the state dictionary before returning it.

    Raises:
        None.
    """
    state = dict(self.__dict__)
    if self.word_tokenizer_type in ["mecab", "sudachi", "jumanpp"]:
        del state["word_tokenizer"]
    return state

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.__init__(vocab_file, spm_file=None, do_lower_case=False, do_word_tokenize=True, do_subword_tokenize=True, word_tokenizer_type='basic', subword_tokenizer_type='wordpiece', never_split=None, unk_token='[UNK]', sep_token='[SEP]', pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', mecab_kwargs=None, sudachi_kwargs=None, jumanpp_kwargs=None, **kwargs)

Initializes a new instance of the BertJapaneseTokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
vocab_file	The path to the vocabulary file. If not using a 'sentencepiece' subword tokenizer, this file is required. TYPE: str
spm_file	The path to the SentencePiece model file. Defaults to None. TYPE: str DEFAULT: None
do_lower_case	A flag to indicate whether the tokenizer should convert all characters to lowercase. Defaults to False. TYPE: bool DEFAULT: False
do_word_tokenize	A flag to indicate whether word tokenization should be performed. Defaults to True. TYPE: bool DEFAULT: True
do_subword_tokenize	A flag to indicate whether subword tokenization should be performed. Defaults to True. TYPE: bool DEFAULT: True
word_tokenizer_type	The type of word tokenizer to use. Must be one of 'basic', 'mecab', 'sudachi', or 'jumanpp'. TYPE: str DEFAULT: 'basic'
subword_tokenizer_type	The type of subword tokenizer to use. Must be one of 'wordpiece', 'character', or 'sentencepiece'. TYPE: str DEFAULT: 'wordpiece'
never_split	A list of tokens that should not be split during tokenization. Defaults to None. TYPE: list DEFAULT: None
unk_token	The token to represent unknown words. Defaults to '[UNK]'. TYPE: str DEFAULT: '[UNK]'
sep_token	The separator token. Defaults to '[SEP]'. TYPE: str DEFAULT: '[SEP]'
pad_token	The padding token. Defaults to '[PAD]'. TYPE: str DEFAULT: '[PAD]'
cls_token	The classification token. Defaults to '[CLS]'. TYPE: str DEFAULT: '[CLS]'
mask_token	The mask token. Defaults to '[MASK]'. TYPE: str DEFAULT: '[MASK]'
mecab_kwargs	Additional keyword arguments for the Mecab word tokenizer. Defaults to None. TYPE: dict DEFAULT: None
sudachi_kwargs	Additional keyword arguments for the Sudachi word tokenizer. Defaults to None. TYPE: dict DEFAULT: None
jumanpp_kwargs	Additional keyword arguments for the Jumanpp word tokenizer. Defaults to None. TYPE: dict DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the specified vocabulary or SentencePiece model file cannot be found, or if an invalid tokenizer type is specified.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def __init__(
    self,
    vocab_file,
    spm_file=None,
    do_lower_case=False,
    do_word_tokenize=True,
    do_subword_tokenize=True,
    word_tokenizer_type="basic",
    subword_tokenizer_type="wordpiece",
    never_split=None,
    unk_token="[UNK]",
    sep_token="[SEP]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    mask_token="[MASK]",
    mecab_kwargs=None,
    sudachi_kwargs=None,
    jumanpp_kwargs=None,
    **kwargs,
):
    """
    Initializes a new instance of the BertJapaneseTokenizer class.

    Args:
        self (object): The instance of the class.
        vocab_file (str): The path to the vocabulary file. If not using a 'sentencepiece' subword tokenizer, this file is required.
        spm_file (str, optional): The path to the SentencePiece model file. Defaults to None.
        do_lower_case (bool): A flag to indicate whether the tokenizer should convert all characters to lowercase. Defaults to False.
        do_word_tokenize (bool): A flag to indicate whether word tokenization should be performed. Defaults to True.
        do_subword_tokenize (bool): A flag to indicate whether subword tokenization should be performed. Defaults to True.
        word_tokenizer_type (str): The type of word tokenizer to use. Must be one of 'basic', 'mecab', 'sudachi', or 'jumanpp'.
        subword_tokenizer_type (str): The type of subword tokenizer to use. Must be one of 'wordpiece', 'character', or 'sentencepiece'.
        never_split (list): A list of tokens that should not be split during tokenization. Defaults to None.
        unk_token (str): The token to represent unknown words. Defaults to '[UNK]'.
        sep_token (str): The separator token. Defaults to '[SEP]'.
        pad_token (str): The padding token. Defaults to '[PAD]'.
        cls_token (str): The classification token. Defaults to '[CLS]'.
        mask_token (str): The mask token. Defaults to '[MASK]'.
        mecab_kwargs (dict): Additional keyword arguments for the Mecab word tokenizer. Defaults to None.
        sudachi_kwargs (dict): Additional keyword arguments for the Sudachi word tokenizer. Defaults to None.
        jumanpp_kwargs (dict): Additional keyword arguments for the Jumanpp word tokenizer. Defaults to None.

    Returns:
        None.

    Raises:
        ValueError: If the specified vocabulary or SentencePiece model file cannot be found, or if an invalid tokenizer type is specified.
    """
    if subword_tokenizer_type == "sentencepiece":
        if not os.path.isfile(spm_file):
            raise ValueError(
                f"Can't find a vocabulary file at path '{spm_file}'. To load the vocabulary from a Google"
                " pretrained model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
            )
        self.spm_file = spm_file
    else:
        if not os.path.isfile(vocab_file):
            raise ValueError(
                f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google"
                " pretrained model use `tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
            )
        self.vocab = load_vocab(vocab_file)
        self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])

    self.do_word_tokenize = do_word_tokenize
    self.word_tokenizer_type = word_tokenizer_type
    self.lower_case = do_lower_case
    self.never_split = never_split
    self.mecab_kwargs = copy.deepcopy(mecab_kwargs)
    self.sudachi_kwargs = copy.deepcopy(sudachi_kwargs)
    self.jumanpp_kwargs = copy.deepcopy(jumanpp_kwargs)
    if do_word_tokenize:
        if word_tokenizer_type == "basic":
            self.word_tokenizer = BasicTokenizer(
                do_lower_case=do_lower_case, never_split=never_split, tokenize_chinese_chars=False
            )
        elif word_tokenizer_type == "mecab":
            self.word_tokenizer = MecabTokenizer(
                do_lower_case=do_lower_case, never_split=never_split, **(mecab_kwargs or {})
            )
        elif word_tokenizer_type == "sudachi":
            self.word_tokenizer = SudachiTokenizer(
                do_lower_case=do_lower_case, never_split=never_split, **(sudachi_kwargs or {})
            )
        elif word_tokenizer_type == "jumanpp":
            self.word_tokenizer = JumanppTokenizer(
                do_lower_case=do_lower_case, never_split=never_split, **(jumanpp_kwargs or {})
            )
        else:
            raise ValueError(f"Invalid word_tokenizer_type '{word_tokenizer_type}' is specified.")

    self.do_subword_tokenize = do_subword_tokenize
    self.subword_tokenizer_type = subword_tokenizer_type
    if do_subword_tokenize:
        if subword_tokenizer_type == "wordpiece":
            self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=str(unk_token))
        elif subword_tokenizer_type == "character":
            self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, unk_token=str(unk_token))
        elif subword_tokenizer_type == "sentencepiece":
            self.subword_tokenizer = SentencepieceTokenizer(vocab=self.spm_file, unk_token=str(unk_token))
        else:
            raise ValueError(f"Invalid subword_tokenizer_type '{subword_tokenizer_type}' is specified.")
    super().__init__(
        spm_file=spm_file,
        unk_token=unk_token,
        sep_token=sep_token,
        pad_token=pad_token,
        cls_token=cls_token,
        mask_token=mask_token,
        do_lower_case=do_lower_case,
        do_word_tokenize=do_word_tokenize,
        do_subword_tokenize=do_subword_tokenize,
        word_tokenizer_type=word_tokenizer_type,
        subword_tokenizer_type=subword_tokenizer_type,
        never_split=never_split,
        mecab_kwargs=mecab_kwargs,
        sudachi_kwargs=sudachi_kwargs,
        jumanpp_kwargs=jumanpp_kwargs,
        **kwargs,
    )

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.__setstate__(state)

PARAMETER	DESCRIPTION
self	The instance of the BertJapaneseTokenizer class. TYPE: BertJapaneseTokenizer
state	The state dictionary containing the attributes to be restored. TYPE: dict

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def __setstate__(self, state):
    """
    Args:
        self (BertJapaneseTokenizer): The instance of the BertJapaneseTokenizer class.
        state (dict): The state dictionary containing the attributes to be restored.

    Returns:
        None.

    Raises:
        None
    """
    self.__dict__ = state
    if self.word_tokenizer_type == "mecab":
        self.word_tokenizer = MecabTokenizer(
            do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.mecab_kwargs or {})
        )
    elif self.word_tokenizer_type == "sudachi":
        self.word_tokenizer = SudachiTokenizer(
            do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.sudachi_kwargs or {})
        )
    elif self.word_tokenizer_type == "jumanpp":
        self.word_tokenizer = JumanppTokenizer(
            do_lower_case=self.do_lower_case, never_split=self.never_split, **(self.jumanpp_kwargs or {})
        )

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A BERT sequence has the following format:

• single sequence: [CLS] X [SEP]
• pair of sequences: [CLS] A [SEP] B [SEP]

PARAMETER	DESCRIPTION
token_ids_0	List of IDs to which the special tokens will be added. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of input IDs with the appropriate special tokens.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def build_inputs_with_special_tokens(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
    adding special tokens. A BERT sequence has the following format:

    - single sequence: `[CLS] X [SEP]`
    - pair of sequences: `[CLS] A [SEP] B [SEP]`

    Args:
        token_ids_0 (`List[int]`):
            List of IDs to which the special tokens will be added.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
    """
    if token_ids_1 is None:
        return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
    cls = [self.cls_token_id]
    sep = [self.sep_token_id]
    return cls + token_ids_0 + sep + token_ids_1 + sep

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    if self.subword_tokenizer_type == "sentencepiece":
        return self.subword_tokenizer.sp_model.decode(tokens)
    out_string = " ".join(tokens).replace(" ##", "").strip()
    return out_string

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

If token_ids_1 is None, this method only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
    pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    sep = [self.sep_token_id]
    cls = [self.cls_token_id]
    if token_ids_1 is None:
        return len(cls + token_ids_0 + sep) * [0]
    return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    if token_ids_1 is not None:
        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
    return [1] + ([0] * len(token_ids_0)) + [1]

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.get_vocab()

This method 'get_vocab' in the class 'BertJapaneseTokenizer' retrieves the vocabulary used by the tokenizer.

PARAMETER	DESCRIPTION
self	The instance of the BertJapaneseTokenizer class. It is a required parameter for instance method access.

RETURNS	DESCRIPTION
	a dictionary representing the vocabulary: If the subword_tokenizer_type is 'sentencepiece', the vocabulary is forwarded by mapping token IDs to their corresponding tokens for the range of 0 to vocab_size. Any added tokens are then added to this vocabulary. If the subword_tokenizer_type is not 'sentencepiece', the vocabulary is a combination of the existing vocabulary and the added_tokens_encoder.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def get_vocab(self):
    """
    This method 'get_vocab' in the class 'BertJapaneseTokenizer' retrieves the vocabulary used by the tokenizer.

    Args:
        self: The instance of the BertJapaneseTokenizer class. It is a required parameter for instance method access.

    Returns:
        a dictionary representing the vocabulary:

            - If the subword_tokenizer_type is 'sentencepiece', the vocabulary is forwarded by mapping token IDs
            to their corresponding tokens for the range of 0 to vocab_size.
            Any added tokens are then added to this vocabulary.
            - If the subword_tokenizer_type is not 'sentencepiece', the vocabulary is a combination of
            the existing vocabulary and the added_tokens_encoder.

    Raises:
        No specific exceptions are documented to be raised by this method.
    """
    if self.subword_tokenizer_type == "sentencepiece":
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab
    return dict(self.vocab, **self.added_tokens_encoder)

mindnlp.transformers.models.bert_japanese.tokenization_bert_japanese.BertJapaneseTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary to a file in the specified directory.

PARAMETER	DESCRIPTION
self	Instance of the BertJapaneseTokenizer class.
save_directory	The directory where the vocabulary file will be saved. TYPE: str
filename_prefix	Optional prefix to be added to the filename. Defaults to None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the path to the saved vocabulary file.

RAISES	DESCRIPTION
FileNotFoundError	If the specified save_directory does not exist.
ValueError	If the subword_tokenizer_type is not supported.
IOError	If there is an issue writing the vocabulary file to the specified location.

Source code in mindnlp/transformers/models/bert_japanese/tokenization_bert_japanese.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary to a file in the specified directory.

    Args:
        self: Instance of the BertJapaneseTokenizer class.
        save_directory (str): The directory where the vocabulary file will be saved.
        filename_prefix (Optional[str]): Optional prefix to be added to the filename. Defaults to None.

    Returns:
        Tuple[str]: A tuple containing the path to the saved vocabulary file.

    Raises:
        FileNotFoundError: If the specified save_directory does not exist.
        ValueError: If the subword_tokenizer_type is not supported.
        IOError: If there is an issue writing the vocabulary file to the specified location.
    """
    if os.path.isdir(save_directory):
        if self.subword_tokenizer_type == "sentencepiece":
            vocab_file = os.path.join(
                save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["spm_file"]
            )
        else:
            vocab_file = os.path.join(
                save_directory,
                (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"],
            )
    else:
        vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory

    if self.subword_tokenizer_type == "sentencepiece":
        with open(vocab_file, "wb") as writer:
            content_spiece_model = self.subword_tokenizer.sp_model.serialized_model_proto()
            writer.write(content_spiece_model)
    else:
        with open(vocab_file, "w", encoding="utf-8") as writer:
            index = 0
            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                        " Please check that the vocabulary is not corrupted!"
                    )
                    index = token_index
                writer.write(token + "\n")
                index += 1
    return (vocab_file,)