codegen

mindnlp.transformers.models.codegen.modeling_codegen.CODEGEN_PRETRAINED_MODEL_ARCHIVE_LIST = ['Salesforce/codegen-350M-nl', 'Salesforce/codegen-350M-multi', 'Salesforce/codegen-350M-mono', 'Salesforce/codegen-2B-nl', 'Salesforce/codegen-2B-multi', 'Salesforce/codegen-2B-mono', 'Salesforce/codegen-6B-nl', 'Salesforce/codegen-6B-multi', 'Salesforce/codegen-6B-mono', 'Salesforce/codegen-16B-nl', 'Salesforce/codegen-16B-multi', 'Salesforce/codegen-16B-mono'] module-attribute

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM

Bases: CodeGenPreTrainedModel

This class represents a code generation model for causal language modeling (LM) using a transformer-based architecture. It inherits from the CodeGenPreTrainedModel class.

ATTRIBUTE	DESCRIPTION
transformer	The transformer model for code generation. TYPE: CodeGenModel
lm_head	The dense layer for predicting the next token in the language modeling task. TYPE: Linear

METHOD	DESCRIPTION
__init__	Initializes the CodeGenForCausalLM instance.
get_output_embeddings	Returns the output embeddings of the LM head.
set_output_embeddings	Sets the output embeddings of the LM head.
prepare_inputs_for_generation	Prepares the input tensors for generation.
forward	Constructs the output of the model for a given set of inputs.
_reorder_cache	Reorders the cache of past key values for beam search or beam sampling.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

class CodeGenForCausalLM(CodeGenPreTrainedModel):

    """
    This class represents a code generation model for causal language modeling (LM) using a transformer-based architecture.
    It inherits from the CodeGenPreTrainedModel class.

    Attributes:
        transformer (CodeGenModel): The transformer model for code generation.
        lm_head (nn.Linear): The dense layer for predicting the next token in the language modeling task.

    Methods:
        __init__: Initializes the CodeGenForCausalLM instance.
        get_output_embeddings: Returns the output embeddings of the LM head.
        set_output_embeddings: Sets the output embeddings of the LM head.
        prepare_inputs_for_generation: Prepares the input tensors for generation.
        forward: Constructs the output of the model for a given set of inputs.
        _reorder_cache: Reorders the cache of past key values for beam search or beam sampling.

    """
    _tied_weights_keys = ["lm_head.weight"]

    def __init__(self, config):
        """
        Initializes an instance of the CodeGenForCausalLM class.

        Args:
            self: The instance of the CodeGenForCausalLM class.
            config: An object containing configuration parameters for the model.

        Returns:
            None.

        Raises:
            This method does not explicitly raise any exceptions.
        """
        super().__init__(config)
        self.transformer = CodeGenModel(config)
        self.lm_head = nn.Linear(config.n_embd, config.vocab_size)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        This method 'get_output_embeddings' in the class 'CodeGenForCausalLM' retrieves the output embeddings from the model's head.

        Args:
            self: The instance of the class 'CodeGenForCausalLM' itself.

        Returns:
            lm_head: This method returns the output embeddings represented by 'self.lm_head', which is of type 'None'.

        Raises:
            None.
        """
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the CodeGenForCausalLM.

        Args:
            self (CodeGenForCausalLM): The instance of the CodeGenForCausalLM class.
            new_embeddings: The new embeddings to be set as the output embeddings.
                It should be of the same type as the current embeddings.

        Returns:
            None.

        Raises:
            None.

        Note:
            This method replaces the current output embeddings of the CodeGenForCausalLM instance
            with the provided new embeddings. The new embeddings should have the same type as the
            current embeddings to ensure compatibility with the other methods of the class.
        """
        self.lm_head = new_embeddings

    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
        """
        Prepare inputs for generation.

        Args:
            self (CodeGenForCausalLM): The instance of the CodeGenForCausalLM class.
            input_ids (tensor): The input tensor containing token IDs for the model.
            past_key_values (tuple or None): Tuple of past key values used for fast decoding.
                Defaults to None.

        Returns:
            dict:
                A dictionary containing the prepared inputs for generation with the following keys:

                - 'input_ids': The trimmed input tensor ready for generation.
                - 'past_key_values': The past_key_values parameter passed to the method.
                - 'use_cache': Boolean flag indicating whether to use caching for faster decoding.
                - 'position_ids': Tensor containing positional IDs for the input tokens.
                - 'attention_mask': Tensor containing attention mask for the input.
                - 'token_type_ids': Tensor containing token type IDs if available.

        Raises:
            None.
        """
        token_type_ids = kwargs.get("token_type_ids", None)
        # Omit tokens covered by past_key_values
        if past_key_values:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]
            if token_type_ids is not None:
                token_type_ids = token_type_ids[:, -input_ids.shape[1] :]

        attention_mask = kwargs.get("attention_mask", None)
        position_ids = kwargs.get("position_ids", None)

        if attention_mask is not None and position_ids is None:
            # create position_ids on the fly for batch generation
            position_ids = attention_mask.long().cumsum(-1) - 1
            position_ids = position_ids.masked_fill(attention_mask == 0, 1)
            if past_key_values:
                position_ids = position_ids[:, -input_ids.shape[1] :]

        return {
            "input_ids": input_ids,
            "past_key_values": past_key_values,
            "use_cache": kwargs.get("use_cache"),
            "position_ids": position_ids,
            "attention_mask": attention_mask,
            "token_type_ids": token_type_ids,
        }

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithPast]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
                `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
                are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        transformer_outputs = self.transformer(
            input_ids,
            past_key_values=past_key_values,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        hidden_states = transformer_outputs[0]

        # make sure sampling in fp16 works correctly and
        # compute loss in fp32 to match with mesh-tf version
        # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
        lm_logits = self.lm_head(hidden_states).to(mindspore.float32)

        loss = None
        if labels is not None:
            # Shift so that tokens < n predict n
            shift_logits = lm_logits[..., :-1, :]
            shift_labels = labels[..., 1:]
            # Flatten the tokens
            loss = ops.cross_entropy(shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1))

            loss = loss.to(hidden_states.dtype)

        if not return_dict:
            output = (lm_logits,) + transformer_outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return CausalLMOutputWithPast(
            loss=loss,
            logits=lm_logits,
            past_key_values=transformer_outputs.past_key_values,
            hidden_states=transformer_outputs.hidden_states,
            attentions=transformer_outputs.attentions,
        )

    @staticmethod
    def _reorder_cache(
        past_key_values: Tuple[Tuple[mindspore.Tensor]], beam_idx: mindspore.Tensor
    ) -> Tuple[Tuple[mindspore.Tensor]]:
        """
        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        """
        return tuple(
            tuple(past_state.index_select(0, beam_idx) for past_state in layer_past)
            for layer_past in past_key_values
        )

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM.__init__(config)

Initializes an instance of the CodeGenForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenForCausalLM class.
config	An object containing configuration parameters for the model.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def __init__(self, config):
    """
    Initializes an instance of the CodeGenForCausalLM class.

    Args:
        self: The instance of the CodeGenForCausalLM class.
        config: An object containing configuration parameters for the model.

    Returns:
        None.

    Raises:
        This method does not explicitly raise any exceptions.
    """
    super().__init__(config)
    self.transformer = CodeGenModel(config)
    self.lm_head = nn.Linear(config.n_embd, config.vocab_size)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM.forward(input_ids=None, past_key_values=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for language modeling. Note that the labels are shifted inside the model, i.e. you can set labels = input_ids Indices are selected in [-100, 0, ..., config.vocab_size] All labels set to -100 are ignored (masked), the loss is only computed for labels in [0, ..., config.vocab_size] TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, CausalLMOutputWithPast]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    transformer_outputs = self.transformer(
        input_ids,
        past_key_values=past_key_values,
        attention_mask=attention_mask,
        token_type_ids=token_type_ids,
        position_ids=position_ids,
        head_mask=head_mask,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    hidden_states = transformer_outputs[0]

    # make sure sampling in fp16 works correctly and
    # compute loss in fp32 to match with mesh-tf version
    # https://github.com/EleutherAI/gpt-neo/blob/89ce74164da2fb16179106f54e2269b5da8db333/models/gpt2/gpt2.py#L179
    lm_logits = self.lm_head(hidden_states).to(mindspore.float32)

    loss = None
    if labels is not None:
        # Shift so that tokens < n predict n
        shift_logits = lm_logits[..., :-1, :]
        shift_labels = labels[..., 1:]
        # Flatten the tokens
        loss = ops.cross_entropy(shift_logits.view(-1, shift_logits.shape[-1]), shift_labels.view(-1))

        loss = loss.to(hidden_states.dtype)

    if not return_dict:
        output = (lm_logits,) + transformer_outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return CausalLMOutputWithPast(
        loss=loss,
        logits=lm_logits,
        past_key_values=transformer_outputs.past_key_values,
        hidden_states=transformer_outputs.hidden_states,
        attentions=transformer_outputs.attentions,
    )

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM.get_output_embeddings()

This method 'get_output_embeddings' in the class 'CodeGenForCausalLM' retrieves the output embeddings from the model's head.

PARAMETER	DESCRIPTION
self	The instance of the class 'CodeGenForCausalLM' itself.

RETURNS	DESCRIPTION
lm_head	This method returns the output embeddings represented by 'self.lm_head', which is of type 'None'.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def get_output_embeddings(self):
    """
    This method 'get_output_embeddings' in the class 'CodeGenForCausalLM' retrieves the output embeddings from the model's head.

    Args:
        self: The instance of the class 'CodeGenForCausalLM' itself.

    Returns:
        lm_head: This method returns the output embeddings represented by 'self.lm_head', which is of type 'None'.

    Raises:
        None.
    """
    return self.lm_head

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, **kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenForCausalLM class. TYPE: CodeGenForCausalLM
input_ids	The input tensor containing token IDs for the model. TYPE: tensor
past_key_values	Tuple of past key values used for fast decoding. Defaults to None. TYPE: tuple or None DEFAULT: None

RETURNS

DESCRIPTION

dict

A dictionary containing the prepared inputs for generation with the following keys: 'input_ids': The trimmed input tensor ready for generation. 'past_key_values': The past_key_values parameter passed to the method. 'use_cache': Boolean flag indicating whether to use caching for faster decoding. 'position_ids': Tensor containing positional IDs for the input tokens. 'attention_mask': Tensor containing attention mask for the input. 'token_type_ids': Tensor containing token type IDs if available.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):
    """
    Prepare inputs for generation.

    Args:
        self (CodeGenForCausalLM): The instance of the CodeGenForCausalLM class.
        input_ids (tensor): The input tensor containing token IDs for the model.
        past_key_values (tuple or None): Tuple of past key values used for fast decoding.
            Defaults to None.

    Returns:
        dict:
            A dictionary containing the prepared inputs for generation with the following keys:

            - 'input_ids': The trimmed input tensor ready for generation.
            - 'past_key_values': The past_key_values parameter passed to the method.
            - 'use_cache': Boolean flag indicating whether to use caching for faster decoding.
            - 'position_ids': Tensor containing positional IDs for the input tokens.
            - 'attention_mask': Tensor containing attention mask for the input.
            - 'token_type_ids': Tensor containing token type IDs if available.

    Raises:
        None.
    """
    token_type_ids = kwargs.get("token_type_ids", None)
    # Omit tokens covered by past_key_values
    if past_key_values:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]
        if token_type_ids is not None:
            token_type_ids = token_type_ids[:, -input_ids.shape[1] :]

    attention_mask = kwargs.get("attention_mask", None)
    position_ids = kwargs.get("position_ids", None)

    if attention_mask is not None and position_ids is None:
        # create position_ids on the fly for batch generation
        position_ids = attention_mask.long().cumsum(-1) - 1
        position_ids = position_ids.masked_fill(attention_mask == 0, 1)
        if past_key_values:
            position_ids = position_ids[:, -input_ids.shape[1] :]

    return {
        "input_ids": input_ids,
        "past_key_values": past_key_values,
        "use_cache": kwargs.get("use_cache"),
        "position_ids": position_ids,
        "attention_mask": attention_mask,
        "token_type_ids": token_type_ids,
    }

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenForCausalLM.set_output_embeddings(new_embeddings)

Sets the output embeddings for the CodeGenForCausalLM.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenForCausalLM class. TYPE: CodeGenForCausalLM
new_embeddings	The new embeddings to be set as the output embeddings. It should be of the same type as the current embeddings.

RETURNS	DESCRIPTION
	None.

Note

This method replaces the current output embeddings of the CodeGenForCausalLM instance with the provided new embeddings. The new embeddings should have the same type as the current embeddings to ensure compatibility with the other methods of the class.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the CodeGenForCausalLM.

    Args:
        self (CodeGenForCausalLM): The instance of the CodeGenForCausalLM class.
        new_embeddings: The new embeddings to be set as the output embeddings.
            It should be of the same type as the current embeddings.

    Returns:
        None.

    Raises:
        None.

    Note:
        This method replaces the current output embeddings of the CodeGenForCausalLM instance
        with the provided new embeddings. The new embeddings should have the same type as the
        current embeddings to ensure compatibility with the other methods of the class.
    """
    self.lm_head = new_embeddings

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenModel

Bases: CodeGenPreTrainedModel

The CodeGenModel class is a Python class that represents a code generation model. It is a subclass of CodeGenPreTrainedModel and provides functionality for forwarding, setting and getting input embeddings, and generating code outputs.

ATTRIBUTE	DESCRIPTION
`embed_dim`	An integer representing the embedding dimension.
`vocab_size`	An integer representing the size of the vocabulary.
`wte`	An embedding layer that maps input tokens to their corresponding embeddings.
`drop`	A dropout layer for regularization.
`h`	A list of CodeGenBlock instances representing code generation blocks.
`ln_f`	A layer normalization layer.
`rotary_dim`	An integer representing the dimension of the rotary encoding.
`gradient_checkpointing`	A boolean indicating whether gradient checkpointing is enabled.

Note

Please refer to the CodeGenPreTrainedModel class for additional inherited attributes and methods.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

class CodeGenModel(CodeGenPreTrainedModel):

    """
    The `CodeGenModel` class is a Python class that represents a code generation model.
    It is a subclass of `CodeGenPreTrainedModel` and provides functionality for forwarding, setting and getting input
    embeddings, and generating code outputs.

    Attributes:
        `embed_dim`: An integer representing the embedding dimension.
        `vocab_size`: An integer representing the size of the vocabulary.
        `wte`: An embedding layer that maps input tokens to their corresponding embeddings.
        `drop`: A dropout layer for regularization.
        `h`: A list of `CodeGenBlock` instances representing code generation blocks.
        `ln_f`: A layer normalization layer.
        `rotary_dim`: An integer representing the dimension of the rotary encoding.
        `gradient_checkpointing`: A boolean indicating whether gradient checkpointing is enabled.

    Note:
        Please refer to the `CodeGenPreTrainedModel` class for additional inherited attributes and methods.

    """
    def __init__(self, config):
        """Initializes an instance of the CodeGenModel class.

        Args:
            self: The instance of the class.
            config:
                An object of the configuration class containing the following attributes:

                - n_embd (int): The embedding dimension.
                - vocab_size (int): The size of the vocabulary.
                - embd_pdrop (float): The dropout probability for the embeddings.
                - n_layer (int): The number of code generation blocks.
                - layer_norm_epsilon (float): The epsilon value for layer normalization.
                - rotary_dim (int): The dimension of the rotary positional embeddings.
                - n_ctx (int): The context length.
                - num_attention_heads (int): The number of attention heads.
            The config object is used to initialize various attributes of the CodeGenModel instance.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)

        self.embed_dim = config.n_embd
        self.vocab_size = config.vocab_size
        self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
        self.drop = nn.Dropout(p=config.embd_pdrop)
        self.h = nn.ModuleList([CodeGenBlock(config) for _ in range(config.n_layer)])
        self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
        self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)

        self.gradient_checkpointing = False

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        This method is part of the CodeGenModel class and is named get_input_embeddings.
        It takes 1 parameter, self, which refers to the instance of the class.

        Args:
            self (CodeGenModel): The instance of the CodeGenModel class.

        Returns:
            None.

        Raises:
            This method does not raise any exceptions.
        """
        return self.wte

    def set_input_embeddings(self, new_embeddings):
        """
        Sets the input embeddings for the CodeGenModel.

        Args:
            self (CodeGenModel): The instance of the CodeGenModel class.
            new_embeddings: The new embeddings to be set. This argument could be of any type.

        Returns:
            None.

        Raises:
            None.
        """
        self.wte = new_embeddings

    def forward(
        self,
        input_ids: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        token_type_ids: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, BaseModelOutputWithPast]:
        """
        Constructs the CodeGenModel.

        Args:
            self (CodeGenModel): The instance of the CodeGenModel class.
            input_ids (Optional[mindspore.Tensor]): The input IDs of the model. Default is None.
            past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): The past key values for the model. Default is None.
            attention_mask (Optional[mindspore.Tensor]): The attention mask for the model. Default is None.
            token_type_ids (Optional[mindspore.Tensor]): The token type IDs for the model. Default is None.
            position_ids (Optional[mindspore.Tensor]): The position IDs for the model. Default is None.
            head_mask (Optional[mindspore.Tensor]): The head mask for the model. Default is None.
            inputs_embeds (Optional[mindspore.Tensor]): The embedded inputs for the model. Default is None.
            use_cache (Optional[bool]): Whether to use cache. Default is None.
            output_attentions (Optional[bool]): Whether to output attentions. Default is None.
            output_hidden_states (Optional[bool]): Whether to output hidden states. Default is None.
            return_dict (Optional[bool]): Whether to return the result as a dictionary. Default is None.

        Returns:
            Union[Tuple, BaseModelOutputWithPast]: The forwarded model output.

        Raises:
            ValueError: If both input_ids and inputs_embeds are specified.
            ValueError: If neither input_ids nor inputs_embeds are specified.
            ValueError: If batch_size is less than or equal to zero.
            Warning: If use_cache is True and config.gradient_checkpointing is True.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.shape
            input_ids = input_ids.view(-1, input_shape[-1])
            batch_size = input_ids.shape[0]
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
            batch_size = inputs_embeds.shape[0]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if token_type_ids is not None:
            token_type_ids = token_type_ids.view(-1, input_shape[-1])

        if past_key_values is None:
            past_length = 0
            past_key_values = tuple([None] * len(self.h))
        else:
            past_length = past_key_values[0][0].shape[-2]

        if position_ids is None:
            position_ids = ops.arange(past_length, input_shape[-1] + past_length, dtype=mindspore.int64)
            position_ids = position_ids.unsqueeze(0)

        # Attention mask.
        if attention_mask is not None:
            if batch_size <= 0:
                raise ValueError("batch_size has to be defined and > 0")
            attention_mask = attention_mask.view(batch_size, -1)
            # We create a 3D attention mask from a 2D tensor mask.
            # Sizes are [batch_size, 1, 1, to_seq_length]
            # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
            # this attention mask is more simple than the triangular masking of causal attention
            # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
            attention_mask = attention_mask[:, None, None, :]

            # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
            # masked positions, this operation will create a tensor which is 0.0 for
            # positions we want to attend and the dtype's smallest value for masked positions.
            # Since we are adding it to the raw scores before the softmax, this is
            # effectively the same as removing these entirely.
            attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
            attention_mask = (1.0 - attention_mask) * finfo(self.dtype, 'min')

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x num_attention_heads x N x N
        # head_mask has shape n_layer x batch x num_attention_heads x N x N
        head_mask = self.get_head_mask(head_mask, self.config.n_layer)

        if inputs_embeds is None:
            inputs_embeds = self.wte(input_ids)

        hidden_states = inputs_embeds

        if token_type_ids is not None:
            token_type_embeds = self.wte(token_type_ids)
            hidden_states = hidden_states + token_type_embeds

        hidden_states = self.drop(hidden_states)

        output_shape = input_shape + (hidden_states.shape[-1],)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
                    "`use_cache=False`..."
                )
                use_cache = False

        presents = () if use_cache else None
        all_self_attentions = () if output_attentions else None
        all_hidden_states = () if output_hidden_states else None
        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            outputs = block(
                hidden_states=hidden_states,
                layer_past=layer_past,
                attention_mask=attention_mask,
                position_ids=position_ids,
                head_mask=head_mask[i],
                use_cache=use_cache,
                output_attentions=output_attentions,
            )

            hidden_states = outputs[0]
            if use_cache is True:
                presents = presents + (outputs[1],)

            if output_attentions:
                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)

        hidden_states = self.ln_f(hidden_states)

        hidden_states = hidden_states.view(output_shape)
        # Add last hidden state
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)

        return BaseModelOutputWithPast(
            last_hidden_state=hidden_states,
            past_key_values=presents,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
        )

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenModel.__init__(config)

Initializes an instance of the CodeGenModel class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of the configuration class containing the following attributes: n_embd (int): The embedding dimension. vocab_size (int): The size of the vocabulary. embd_pdrop (float): The dropout probability for the embeddings. n_layer (int): The number of code generation blocks. layer_norm_epsilon (float): The epsilon value for layer normalization. rotary_dim (int): The dimension of the rotary positional embeddings. n_ctx (int): The context length. num_attention_heads (int): The number of attention heads.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def __init__(self, config):
    """Initializes an instance of the CodeGenModel class.

    Args:
        self: The instance of the class.
        config:
            An object of the configuration class containing the following attributes:

            - n_embd (int): The embedding dimension.
            - vocab_size (int): The size of the vocabulary.
            - embd_pdrop (float): The dropout probability for the embeddings.
            - n_layer (int): The number of code generation blocks.
            - layer_norm_epsilon (float): The epsilon value for layer normalization.
            - rotary_dim (int): The dimension of the rotary positional embeddings.
            - n_ctx (int): The context length.
            - num_attention_heads (int): The number of attention heads.
        The config object is used to initialize various attributes of the CodeGenModel instance.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)

    self.embed_dim = config.n_embd
    self.vocab_size = config.vocab_size
    self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
    self.drop = nn.Dropout(p=config.embd_pdrop)
    self.h = nn.ModuleList([CodeGenBlock(config) for _ in range(config.n_layer)])
    self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
    self.rotary_dim = min(config.rotary_dim, config.n_ctx // config.num_attention_heads)

    self.gradient_checkpointing = False

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenModel.forward(input_ids=None, past_key_values=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

Constructs the CodeGenModel.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenModel class. TYPE: CodeGenModel
input_ids	The input IDs of the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
past_key_values	The past key values for the model. Default is None. TYPE: Optional[Tuple[Tuple[Tensor]]] DEFAULT: None
attention_mask	The attention mask for the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
token_type_ids	The token type IDs for the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
position_ids	The position IDs for the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
head_mask	The head mask for the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
inputs_embeds	The embedded inputs for the model. Default is None. TYPE: Optional[Tensor] DEFAULT: None
use_cache	Whether to use cache. Default is None. TYPE: Optional[bool] DEFAULT: None
output_attentions	Whether to output attentions. Default is None. TYPE: Optional[bool] DEFAULT: None
output_hidden_states	Whether to output hidden states. Default is None. TYPE: Optional[bool] DEFAULT: None
return_dict	Whether to return the result as a dictionary. Default is None. TYPE: Optional[bool] DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, BaseModelOutputWithPast]	Union[Tuple, BaseModelOutputWithPast]: The forwarded model output.

RAISES	DESCRIPTION
ValueError	If both input_ids and inputs_embeds are specified.
ValueError	If neither input_ids nor inputs_embeds are specified.
ValueError	If batch_size is less than or equal to zero.
Warning	If use_cache is True and config.gradient_checkpointing is True.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def forward(
    self,
    input_ids: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    token_type_ids: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, BaseModelOutputWithPast]:
    """
    Constructs the CodeGenModel.

    Args:
        self (CodeGenModel): The instance of the CodeGenModel class.
        input_ids (Optional[mindspore.Tensor]): The input IDs of the model. Default is None.
        past_key_values (Optional[Tuple[Tuple[mindspore.Tensor]]]): The past key values for the model. Default is None.
        attention_mask (Optional[mindspore.Tensor]): The attention mask for the model. Default is None.
        token_type_ids (Optional[mindspore.Tensor]): The token type IDs for the model. Default is None.
        position_ids (Optional[mindspore.Tensor]): The position IDs for the model. Default is None.
        head_mask (Optional[mindspore.Tensor]): The head mask for the model. Default is None.
        inputs_embeds (Optional[mindspore.Tensor]): The embedded inputs for the model. Default is None.
        use_cache (Optional[bool]): Whether to use cache. Default is None.
        output_attentions (Optional[bool]): Whether to output attentions. Default is None.
        output_hidden_states (Optional[bool]): Whether to output hidden states. Default is None.
        return_dict (Optional[bool]): Whether to return the result as a dictionary. Default is None.

    Returns:
        Union[Tuple, BaseModelOutputWithPast]: The forwarded model output.

    Raises:
        ValueError: If both input_ids and inputs_embeds are specified.
        ValueError: If neither input_ids nor inputs_embeds are specified.
        ValueError: If batch_size is less than or equal to zero.
        Warning: If use_cache is True and config.gradient_checkpointing is True.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
        input_shape = input_ids.shape
        input_ids = input_ids.view(-1, input_shape[-1])
        batch_size = input_ids.shape[0]
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
        batch_size = inputs_embeds.shape[0]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    if token_type_ids is not None:
        token_type_ids = token_type_ids.view(-1, input_shape[-1])

    if past_key_values is None:
        past_length = 0
        past_key_values = tuple([None] * len(self.h))
    else:
        past_length = past_key_values[0][0].shape[-2]

    if position_ids is None:
        position_ids = ops.arange(past_length, input_shape[-1] + past_length, dtype=mindspore.int64)
        position_ids = position_ids.unsqueeze(0)

    # Attention mask.
    if attention_mask is not None:
        if batch_size <= 0:
            raise ValueError("batch_size has to be defined and > 0")
        attention_mask = attention_mask.view(batch_size, -1)
        # We create a 3D attention mask from a 2D tensor mask.
        # Sizes are [batch_size, 1, 1, to_seq_length]
        # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
        # this attention mask is more simple than the triangular masking of causal attention
        # used in OpenAI GPT, we just need to prepare the broadcast dimension here.
        attention_mask = attention_mask[:, None, None, :]

        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
        # masked positions, this operation will create a tensor which is 0.0 for
        # positions we want to attend and the dtype's smallest value for masked positions.
        # Since we are adding it to the raw scores before the softmax, this is
        # effectively the same as removing these entirely.
        attention_mask = attention_mask.to(dtype=self.dtype)  # fp16 compatibility
        attention_mask = (1.0 - attention_mask) * finfo(self.dtype, 'min')

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x num_attention_heads x N x N
    # head_mask has shape n_layer x batch x num_attention_heads x N x N
    head_mask = self.get_head_mask(head_mask, self.config.n_layer)

    if inputs_embeds is None:
        inputs_embeds = self.wte(input_ids)

    hidden_states = inputs_embeds

    if token_type_ids is not None:
        token_type_embeds = self.wte(token_type_ids)
        hidden_states = hidden_states + token_type_embeds

    hidden_states = self.drop(hidden_states)

    output_shape = input_shape + (hidden_states.shape[-1],)

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting "
                "`use_cache=False`..."
            )
            use_cache = False

    presents = () if use_cache else None
    all_self_attentions = () if output_attentions else None
    all_hidden_states = () if output_hidden_states else None
    for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        outputs = block(
            hidden_states=hidden_states,
            layer_past=layer_past,
            attention_mask=attention_mask,
            position_ids=position_ids,
            head_mask=head_mask[i],
            use_cache=use_cache,
            output_attentions=output_attentions,
        )

        hidden_states = outputs[0]
        if use_cache is True:
            presents = presents + (outputs[1],)

        if output_attentions:
            all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)

    hidden_states = self.ln_f(hidden_states)

    hidden_states = hidden_states.view(output_shape)
    # Add last hidden state
    if output_hidden_states:
        all_hidden_states = all_hidden_states + (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)

    return BaseModelOutputWithPast(
        last_hidden_state=hidden_states,
        past_key_values=presents,
        hidden_states=all_hidden_states,
        attentions=all_self_attentions,
    )

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenModel.get_input_embeddings()

This method is part of the CodeGenModel class and is named get_input_embeddings. It takes 1 parameter, self, which refers to the instance of the class.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenModel class. TYPE: CodeGenModel

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def get_input_embeddings(self):
    """
    This method is part of the CodeGenModel class and is named get_input_embeddings.
    It takes 1 parameter, self, which refers to the instance of the class.

    Args:
        self (CodeGenModel): The instance of the CodeGenModel class.

    Returns:
        None.

    Raises:
        This method does not raise any exceptions.
    """
    return self.wte

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenModel.set_input_embeddings(new_embeddings)

Sets the input embeddings for the CodeGenModel.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenModel class. TYPE: CodeGenModel
new_embeddings	The new embeddings to be set. This argument could be of any type.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

def set_input_embeddings(self, new_embeddings):
    """
    Sets the input embeddings for the CodeGenModel.

    Args:
        self (CodeGenModel): The instance of the CodeGenModel class.
        new_embeddings: The new embeddings to be set. This argument could be of any type.

    Returns:
        None.

    Raises:
        None.
    """
    self.wte = new_embeddings

mindnlp.transformers.models.codegen.modeling_codegen.CodeGenPreTrainedModel

Bases: PreTrainedModel

An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.

Source code in mindnlp/transformers/models/codegen/modeling_codegen.py

class CodeGenPreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """
    config_class = CodeGenConfig
    base_model_prefix = "transformer"
    _no_split_modules = ["CodeGenBlock"]

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(initializer(Normal(self.config.initializer_range),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.bias:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, self.config.initializer_range, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))
        elif isinstance(cell, nn.LayerNorm):
            cell.weight.set_data(initializer('ones', cell.weight.shape, cell.weight.dtype))
            cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))

mindnlp.transformers.models.codegen.configuration_codegen.CodeGenConfig

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [CodeGenModel]. It is used to instantiate a CodeGen model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the CodeGen Salesforce/codegen-2B-mono architecture. Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the CodeGen model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [CodeGenModel]. TYPE: `int`, *optional*, defaults to 50400 DEFAULT: 50400
n_positions	The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). TYPE: `int`, *optional*, defaults to 2048 DEFAULT: 2048
n_ctx	This attribute is used in CodeGenModel.__init__ without any real effect. TYPE: `int`, *optional*, defaults to 2048 DEFAULT: 2048
n_embd	Dimensionality of the embeddings and hidden states. TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
n_layer	Number of hidden layers in the Transformer encoder. TYPE: `int`, *optional*, defaults to 28 DEFAULT: 28
n_head	Number of attention heads for each attention layer in the Transformer encoder. TYPE: `int`, *optional*, defaults to 16 DEFAULT: 16
rotary_dim	Number of dimensions in the embedding that Rotary Position Embedding is applied to. TYPE: `int`, *optional*, defaults to 64 DEFAULT: 64
n_inner	Dimensionality of the inner feed-forward layers. None will set it to 4 times n_embd TYPE: `int`, *optional* DEFAULT: None
activation_function	Activation function, to be selected in the list ["relu", "silu", "gelu", "tanh", "gelu_new"]. TYPE: `str`, *optional*, defaults to `"gelu_new"` DEFAULT: 'gelu_new'
resid_pdrop	The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
embd_pdrop	The dropout ratio for the embeddings. TYPE: `int`, *optional*, defaults to 0.0 DEFAULT: 0.0
attn_pdrop	The dropout ratio for the attention. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0
layer_norm_epsilon	The epsilon to use in the layer normalization layers. TYPE: `float`, *optional*, defaults to 1e-05 DEFAULT: 1e-05
initializer_range	The standard deviation of the truncated_normal_initializer for initializing all weight matrices. TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
bos_token_id	Beginning of stream token id. TYPE: `int`, *optional*, defaults to 50256 DEFAULT: 50256
eos_token_id	End of stream token id. TYPE: `int`, *optional*, defaults to 50256 DEFAULT: 50256
tie_word_embeddings	Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the model has a output word embedding layer. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

Example

>>> from transformers import CodeGenConfig, CodeGenModel
...
>>> # Initializing a CodeGen 6B configuration
>>> configuration = CodeGenConfig()
...
>>> # Initializing a model (with random weights) from the configuration
>>> model = CodeGenModel(configuration)
...
>>> # Accessing the model configuration
>>> configuration = model.config

Source code in mindnlp/transformers/models/codegen/configuration_codegen.py

class CodeGenConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`CodeGenModel`]. It is used to instantiate a
    CodeGen model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the CodeGen
    [Salesforce/codegen-2B-mono](https://hf-mirror.com/Salesforce/codegen-2B-mono) architecture. Configuration objects
    inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from
    [`PretrainedConfig`] for more information.

    Args:
        vocab_size (`int`, *optional*, defaults to 50400):
            Vocabulary size of the CodeGen model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`CodeGenModel`].
        n_positions (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with. Typically set this to something large
            just in case (e.g., 512 or 1024 or 2048).
        n_ctx (`int`, *optional*, defaults to 2048):
            This attribute is used in `CodeGenModel.__init__` without any real effect.
        n_embd (`int`, *optional*, defaults to 4096):
            Dimensionality of the embeddings and hidden states.
        n_layer (`int`, *optional*, defaults to 28):
            Number of hidden layers in the Transformer encoder.
        n_head (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer encoder.
        rotary_dim (`int`, *optional*, defaults to 64):
            Number of dimensions in the embedding that Rotary Position Embedding is applied to.
        n_inner (`int`, *optional*):
            Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd
        activation_function (`str`, *optional*, defaults to `"gelu_new"`):
            Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new"]`.
        resid_pdrop (`float`, *optional*, defaults to 0.0):
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
        embd_pdrop (`int`, *optional*, defaults to 0.0):
            The dropout ratio for the embeddings.
        attn_pdrop (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention.
        layer_norm_epsilon (`float`, *optional*, defaults to 1e-05):
            The epsilon to use in the layer normalization layers.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
        bos_token_id (`int`, *optional*, defaults to 50256):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 50256):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
            model has a output word embedding layer.

    Example:
        ```python
        >>> from transformers import CodeGenConfig, CodeGenModel
        ...
        >>> # Initializing a CodeGen 6B configuration
        >>> configuration = CodeGenConfig()
        ...
        >>> # Initializing a model (with random weights) from the configuration
        >>> model = CodeGenModel(configuration)
        ...
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "codegen"
    attribute_map = {
        "max_position_embeddings": "n_positions",
        "hidden_size": "n_embd",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=50400,
        n_positions=2048,
        n_ctx=2048,
        n_embd=4096,
        n_layer=28,
        n_head=16,
        rotary_dim=64,
        n_inner=None,
        activation_function="gelu_new",
        resid_pdrop=0.0,
        embd_pdrop=0.0,
        attn_pdrop=0.0,
        layer_norm_epsilon=1e-5,
        initializer_range=0.02,
        use_cache=True,
        bos_token_id=50256,
        eos_token_id=50256,
        tie_word_embeddings=False,
        **kwargs,
    ):
        """
        Initializes an instance of the CodeGenConfig class.

        Args:
            self: The instance of the class.
            vocab_size (int, optional): The size of the vocabulary. Defaults to 50400.
            n_positions (int, optional): The number of positions. Defaults to 2048.
            n_ctx (int, optional): The context size. Defaults to 2048.
            n_embd (int, optional): The embedding size. Defaults to 4096.
            n_layer (int, optional): The number of layers. Defaults to 28.
            n_head (int, optional): The number of attention heads. Defaults to 16.
            rotary_dim (int, optional): The dimension for rotary positional embeddings. Defaults to 64.
            n_inner (int, optional): The inner size of the feed-forward networks. Defaults to None.
            activation_function (str, optional): The activation function to use. Defaults to 'gelu_new'.
            resid_pdrop (float, optional): The dropout rate for residual connections. Defaults to 0.0.
            embd_pdrop (float, optional): The dropout rate for embeddings. Defaults to 0.0.
            attn_pdrop (float, optional): The dropout rate for attention probabilities. Defaults to 0.0.
            layer_norm_epsilon (float, optional): The epsilon value for layer normalization. Defaults to 1e-05.
            initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
            use_cache (bool, optional): Whether to use caching. Defaults to True.
            bos_token_id (int, optional): The ID of the beginning-of-sentence token. Defaults to 50256.
            eos_token_id (int, optional): The ID of the end-of-sentence token. Defaults to 50256.
            tie_word_embeddings (bool, optional): Whether to tie word embeddings. Defaults to False.

        Returns:
            None.

        Raises:
            None.
        """
        self.vocab_size = vocab_size
        self.n_ctx = n_ctx
        self.n_positions = n_positions
        self.n_embd = n_embd
        self.n_layer = n_layer
        self.n_head = n_head
        self.n_inner = n_inner
        self.rotary_dim = rotary_dim
        self.activation_function = activation_function
        self.resid_pdrop = resid_pdrop
        self.embd_pdrop = embd_pdrop
        self.attn_pdrop = attn_pdrop
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_range = initializer_range
        self.use_cache = use_cache

        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id

        super().__init__(
            bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
        )

mindnlp.transformers.models.codegen.configuration_codegen.CodeGenConfig.__init__(vocab_size=50400, n_positions=2048, n_ctx=2048, n_embd=4096, n_layer=28, n_head=16, rotary_dim=64, n_inner=None, activation_function='gelu_new', resid_pdrop=0.0, embd_pdrop=0.0, attn_pdrop=0.0, layer_norm_epsilon=1e-05, initializer_range=0.02, use_cache=True, bos_token_id=50256, eos_token_id=50256, tie_word_embeddings=False, **kwargs)

Initializes an instance of the CodeGenConfig class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_size	The size of the vocabulary. Defaults to 50400. TYPE: int DEFAULT: 50400
n_positions	The number of positions. Defaults to 2048. TYPE: int DEFAULT: 2048
n_ctx	The context size. Defaults to 2048. TYPE: int DEFAULT: 2048
n_embd	The embedding size. Defaults to 4096. TYPE: int DEFAULT: 4096
n_layer	The number of layers. Defaults to 28. TYPE: int DEFAULT: 28
n_head	The number of attention heads. Defaults to 16. TYPE: int DEFAULT: 16
rotary_dim	The dimension for rotary positional embeddings. Defaults to 64. TYPE: int DEFAULT: 64
n_inner	The inner size of the feed-forward networks. Defaults to None. TYPE: int DEFAULT: None
activation_function	The activation function to use. Defaults to 'gelu_new'. TYPE: str DEFAULT: 'gelu_new'
resid_pdrop	The dropout rate for residual connections. Defaults to 0.0. TYPE: float DEFAULT: 0.0
embd_pdrop	The dropout rate for embeddings. Defaults to 0.0. TYPE: float DEFAULT: 0.0
attn_pdrop	The dropout rate for attention probabilities. Defaults to 0.0. TYPE: float DEFAULT: 0.0
layer_norm_epsilon	The epsilon value for layer normalization. Defaults to 1e-05. TYPE: float DEFAULT: 1e-05
initializer_range	The range for weight initialization. Defaults to 0.02. TYPE: float DEFAULT: 0.02
use_cache	Whether to use caching. Defaults to True. TYPE: bool DEFAULT: True
bos_token_id	The ID of the beginning-of-sentence token. Defaults to 50256. TYPE: int DEFAULT: 50256
eos_token_id	The ID of the end-of-sentence token. Defaults to 50256. TYPE: int DEFAULT: 50256
tie_word_embeddings	Whether to tie word embeddings. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/configuration_codegen.py

def __init__(
    self,
    vocab_size=50400,
    n_positions=2048,
    n_ctx=2048,
    n_embd=4096,
    n_layer=28,
    n_head=16,
    rotary_dim=64,
    n_inner=None,
    activation_function="gelu_new",
    resid_pdrop=0.0,
    embd_pdrop=0.0,
    attn_pdrop=0.0,
    layer_norm_epsilon=1e-5,
    initializer_range=0.02,
    use_cache=True,
    bos_token_id=50256,
    eos_token_id=50256,
    tie_word_embeddings=False,
    **kwargs,
):
    """
    Initializes an instance of the CodeGenConfig class.

    Args:
        self: The instance of the class.
        vocab_size (int, optional): The size of the vocabulary. Defaults to 50400.
        n_positions (int, optional): The number of positions. Defaults to 2048.
        n_ctx (int, optional): The context size. Defaults to 2048.
        n_embd (int, optional): The embedding size. Defaults to 4096.
        n_layer (int, optional): The number of layers. Defaults to 28.
        n_head (int, optional): The number of attention heads. Defaults to 16.
        rotary_dim (int, optional): The dimension for rotary positional embeddings. Defaults to 64.
        n_inner (int, optional): The inner size of the feed-forward networks. Defaults to None.
        activation_function (str, optional): The activation function to use. Defaults to 'gelu_new'.
        resid_pdrop (float, optional): The dropout rate for residual connections. Defaults to 0.0.
        embd_pdrop (float, optional): The dropout rate for embeddings. Defaults to 0.0.
        attn_pdrop (float, optional): The dropout rate for attention probabilities. Defaults to 0.0.
        layer_norm_epsilon (float, optional): The epsilon value for layer normalization. Defaults to 1e-05.
        initializer_range (float, optional): The range for weight initialization. Defaults to 0.02.
        use_cache (bool, optional): Whether to use caching. Defaults to True.
        bos_token_id (int, optional): The ID of the beginning-of-sentence token. Defaults to 50256.
        eos_token_id (int, optional): The ID of the end-of-sentence token. Defaults to 50256.
        tie_word_embeddings (bool, optional): Whether to tie word embeddings. Defaults to False.

    Returns:
        None.

    Raises:
        None.
    """
    self.vocab_size = vocab_size
    self.n_ctx = n_ctx
    self.n_positions = n_positions
    self.n_embd = n_embd
    self.n_layer = n_layer
    self.n_head = n_head
    self.n_inner = n_inner
    self.rotary_dim = rotary_dim
    self.activation_function = activation_function
    self.resid_pdrop = resid_pdrop
    self.embd_pdrop = embd_pdrop
    self.attn_pdrop = attn_pdrop
    self.layer_norm_epsilon = layer_norm_epsilon
    self.initializer_range = initializer_range
    self.use_cache = use_cache

    self.bos_token_id = bos_token_id
    self.eos_token_id = eos_token_id

    super().__init__(
        bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
    )

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer

Bases: PreTrainedTokenizer

Construct a CodeGen tokenizer. Based on byte-level Byte-Pair-Encoding.

This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not:

Example

>>> from transformers import CodeGenTokenizer
...
>>> tokenizer = CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono")
>>> tokenizer("Hello world")["input_ids"]
[15496, 995]
>>> tokenizer(" Hello world")["input_ids"]
[18435, 995]

You can get around that behavior by passing add_prefix_space=True when instantiating this tokenizer or when you call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.

When used with is_split_into_words=True, this tokenizer will add a space before each word (even the first one).

This tokenizer inherits from [PreTrainedTokenizer] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`
merges_file	Path to the merges file. TYPE: `str`
errors	Paradigm to follow when decoding bytes to UTF-8. See bytes.decode for more information. TYPE: `str`, *optional*, defaults to `"replace"` DEFAULT: 'replace'
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
bos_token	The beginning of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
eos_token	The end of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
pad_token	The token used for padding, for example when batching sequences of different lengths. TYPE: `str`, *optional* DEFAULT: None
add_prefix_space	Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. (CodeGen tokenizer detect beginning of words by the preceding space). TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
add_bos_token	Whether to add a beginning of sequence token at the start of sequences. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

class CodeGenTokenizer(PreTrainedTokenizer):
    """
    Construct a CodeGen tokenizer. Based on byte-level Byte-Pair-Encoding.

    This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will
    be encoded differently whether it is at the beginning of the sentence (without space) or not:

    Example:
        ```python
        >>> from transformers import CodeGenTokenizer
        ...
        >>> tokenizer = CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono")
        >>> tokenizer("Hello world")["input_ids"]
        [15496, 995]
        >>> tokenizer(" Hello world")["input_ids"]
        [18435, 995]
        ```

    You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you
    call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance.

    <Tip>

    When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).

    </Tip>

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        errors (`str`, *optional*, defaults to `"replace"`):
            Paradigm to follow when decoding bytes to UTF-8. See
            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        pad_token (`str`, *optional*):
            The token used for padding, for example when batching sequences of different lengths.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word. (CodeGen tokenizer detect beginning of words by the preceding space).
        add_bos_token (`bool`, *optional*, defaults to `False`):
            Whether to add a beginning of sequence token at the start of sequences.
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        merges_file,
        errors="replace",
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        pad_token=None,
        add_prefix_space=False,
        add_bos_token=False,
        **kwargs,
    ):
        """
        Initializes an instance of the CodeGenTokenizer class.

        Args:
            self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
            vocab_file (str): The path to the vocabulary file.
            merges_file (str): The path to the merges file.
            errors (str, optional): Specifies the error handling scheme. Defaults to 'replace'.
            unk_token (str, optional): The special token used to represent unknown tokens. Defaults to 'endoftext'.
            bos_token (str, optional): The special token used to represent the beginning of a sentence. Defaults to 'endoftext'.
            eos_token (str, optional): The special token used to represent the end of a sentence. Defaults to 'endoftext'.
            pad_token (str, optional): The special token used to represent padding. Defaults to None.
            add_prefix_space (bool, optional): Specifies whether a prefix space should be added. Defaults to False.
            add_bos_token (bool, optional): Specifies whether the bos_token should be added. Defaults to False.

        Returns:
            None.

        Raises:
            FileNotFoundError: If the vocab_file or merges_file could not be found.
            UnicodeDecodeError: If there is an error decoding the vocab_file or merges_file.
            TypeError: If any of the special token parameters (unk_token, bos_token, eos_token, pad_token) are not strings.
            TypeError: If add_bos_token or add_prefix_space are not boolean values.
            ValueError: If the merges_file is not formatted correctly.

        Note:
            This method initializes the CodeGenTokenizer class by loading the vocabulary and merges files,
            setting the error handling scheme, and initializing various attributes.

        Example:
            ```python
            >>> tokenizer = CodeGenTokenizer('vocab.txt', 'merges.txt', errors='ignore', unk_token='<unk>', bos_token='<s>')
            ```
        """
        bos_token = AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
        pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
        self.add_bos_token = add_bos_token

        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.errors = errors  # how to handle errors in decoding
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
        with open(merges_file, encoding="utf-8") as merges_handle:
            bpe_merges = merges_handle.read().split("\n")[1:-1]
        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
        self.cache = {}
        self.add_prefix_space = add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
        super().__init__(
            errors=errors,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            pad_token=pad_token,
            add_prefix_space=add_prefix_space,
            add_bos_token=add_bos_token,
            **kwargs,
        )

    @property
    def vocab_size(self):
        """
        This method returns the size of the vocabulary in the CodeGenTokenizer instance.

        Args:
            self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
                It is used to access the encoder attribute to calculate the vocabulary size.

        Returns:
            int: The size of the vocabulary in the CodeGenTokenizer instance.

        Raises:
            This method does not raise any exceptions.
        """
        return len(self.encoder)

    def get_vocab(self):
        """
        Returns a dictionary containing the vocabulary of the CodeGenTokenizer instance.

        Args:
            self (CodeGenTokenizer): The CodeGenTokenizer instance.

        Returns:
            dict: A dictionary containing the vocabulary of the CodeGenTokenizer. The keys are tokens
                from the encoder and added_tokens_encoder, and the values are their corresponding indices.

        Raises:
            None.
        """
        return dict(self.encoder, **self.added_tokens_encoder)

    def bpe(self, token):
        """
        This method implements the Byte Pair Encoding (BPE) algorithm to tokenize a given token.

        Args:
            self (CodeGenTokenizer): An instance of the CodeGenTokenizer class.
            token (str): The token to be tokenized using the BPE algorithm.

        Returns:
            str: The token after applying the BPE algorithm.

        Raises:
            None.

        This method first checks if the token is already present in the cache and returns the cached value if found.
        It then converts the token into a tuple of characters and generates a list of all possible pairs of characters in the token.
        If no pairs are found, the method returns the original token as there is no need for further processing.

        The method then enters a loop where it iteratively selects the pair with the lowest rank according to the BPE ranks.
        If the selected pair is not present in the BPE ranks, the loop is terminated.

        The method then finds the first occurrence of the selected pair in the token and replaces it with the concatenation of the pair.
        This process is repeated until no more occurrences of the selected pair are found in the token.

        Finally, the token is converted back to a string and stored in the cache for future use before being returned as the result of the method.
        """
        if token in self.cache:
            return self.cache[token]
        word = tuple(token)
        pairs = get_pairs(word)

        if not pairs:
            return token

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            pairs = get_pairs(word)
        word = " ".join(word)
        self.cache[token] = word
        return word

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        """
        Args:
            self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
            token_ids_0 (list): A list of token IDs for the first input sequence.
            token_ids_1 (list, optional): A list of token IDs for the second input sequence. Defaults to None.

        Returns:
            None.

        Raises:
            None.
        """
        if self.add_bos_token:
            bos_token_ids = [self.bos_token_id]
        else:
            bos_token_ids = []

        output = bos_token_ids + token_ids_0

        if token_ids_1 is None:
            return output

        return output + bos_token_ids + token_ids_1

    def _tokenize(self, text):
        """Tokenize a string."""
        bpe_tokens = []
        for token in re.findall(self.pat, text):
            token = "".join(
                self.byte_encoder[b] for b in token.encode("utf-8")
            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
        return bpe_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        return self.decoder.get(index)

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        text = "".join(tokens)
        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
        return text

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the generated vocabulary files to the specified directory.

        Args:
            self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
            save_directory (str): The directory path where the vocabulary files will be saved.
            filename_prefix (Optional[str]): An optional prefix to be added to the filenames. Default is None.

        Returns:
            Tuple[str]: A tuple containing the paths to the saved vocabulary and merge files.

        Raises:
            ValueError: If the provided save_directory is not a valid directory path.
            IOError: If there are issues with writing the vocabulary or merge files to the specified directory.
            RuntimeError: If the BPE merge indices are not consecutive, indicating potential corruption in the tokenizer.
        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )
        merge_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        index = 0
        with open(merge_file, "w", encoding="utf-8") as writer:
            writer.write("#version: 0.2\n")
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning(
                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                        " Please check that the tokenizer is not corrupted!"
                    )
                    index = token_index
                writer.write(" ".join(bpe_tokens) + "\n")
                index += 1

        return vocab_file, merge_file

    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
        """
        Prepare the text for tokenization by adding prefix space if required.

        Args:
            self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
            text (str): The input text to be prepared for tokenization.
            is_split_into_words (bool): A flag indicating if the text is already split into words. Default is False.

        Returns:
            str: The text prepared for tokenization with potential prefix space added.

        Raises:
            KeyError: If 'add_prefix_space' is not found in kwargs.
        """
        add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
        if is_split_into_words or add_prefix_space:
            text = " " + text
        return (text, kwargs)

    def decode(
        self,
        token_ids,
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        truncate_before_pattern: Optional[List[str]] = None,
        **kwargs,
    ) -> str:
        r"""
        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
        tokens and clean up tokenization spaces.
        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.
            clean_up_tokenization_spaces (`bool`, *optional*):
                Whether or not to clean up the tokenization spaces. If `None`, will default to
                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
            truncate_before_pattern (`List[str]`, *optional*, defaults to `None`):
                A list of regular expression strings that will be used to truncate the returned string. This can be
                used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning
                of a new line). An example pattern could be `["^#", re.escape("<|endoftext|>"), "^'''", "\n\n\n"]`.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the underlying model specific decode method.

        Returns:
            `str`: The decoded sentence.
        """
        decoded_text = super()._decode(
            token_ids=token_ids,
            skip_special_tokens=skip_special_tokens,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            **kwargs,
        )

        if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
            decoded_text = self.truncate(decoded_text, truncate_before_pattern)

        return decoded_text

    def truncate(self, completion, truncate_before_pattern):
        """
        This method 'truncate' is defined in the 'CodeGenTokenizer' class and is used to truncate a given completion based on specified patterns.

        Args:
            self: The instance of the class.
            completion (str): The completion string to be truncated.
            truncate_before_pattern (list of str): A list of patterns to truncate the completion string before.

        Returns:
            None.

        Raises:
            None.
        """
        def find_re(string, pattern, start_pos):
            m = pattern.search(string, start_pos)
            return m.start() if m else -1

        terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]

        prints = list(re.finditer("^print", completion, re.MULTILINE))

        if len(prints) > 1:
            completion = completion[: prints[1].start()]

        defs = list(re.finditer("^def", completion, re.MULTILINE))

        if len(defs) > 1:
            completion = completion[: defs[1].start()]

        start_pos = 0

        terminals_pos = [
            pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
        ]

        if len(terminals_pos) > 0:
            return completion[: min(terminals_pos)]
        return completion

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.vocab_size property

This method returns the size of the vocabulary in the CodeGenTokenizer instance.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizer class. It is used to access the encoder attribute to calculate the vocabulary size. TYPE: CodeGenTokenizer

RETURNS	DESCRIPTION
int	The size of the vocabulary in the CodeGenTokenizer instance.

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.__init__(vocab_file, merges_file, errors='replace', unk_token='<|endoftext|>', bos_token='<|endoftext|>', eos_token='<|endoftext|>', pad_token=None, add_prefix_space=False, add_bos_token=False, **kwargs)

Initializes an instance of the CodeGenTokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizer class. TYPE: CodeGenTokenizer
vocab_file	The path to the vocabulary file. TYPE: str
merges_file	The path to the merges file. TYPE: str
errors	Specifies the error handling scheme. Defaults to 'replace'. TYPE: str DEFAULT: 'replace'
unk_token	The special token used to represent unknown tokens. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
bos_token	The special token used to represent the beginning of a sentence. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
eos_token	The special token used to represent the end of a sentence. Defaults to 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
pad_token	The special token used to represent padding. Defaults to None. TYPE: str DEFAULT: None
add_prefix_space	Specifies whether a prefix space should be added. Defaults to False. TYPE: bool DEFAULT: False
add_bos_token	Specifies whether the bos_token should be added. Defaults to False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
FileNotFoundError	If the vocab_file or merges_file could not be found.
UnicodeDecodeError	If there is an error decoding the vocab_file or merges_file.
TypeError	If any of the special token parameters (unk_token, bos_token, eos_token, pad_token) are not strings.
TypeError	If add_bos_token or add_prefix_space are not boolean values.
ValueError	If the merges_file is not formatted correctly.

Note

This method initializes the CodeGenTokenizer class by loading the vocabulary and merges files, setting the error handling scheme, and initializing various attributes.

Example

>>> tokenizer = CodeGenTokenizer('vocab.txt', 'merges.txt', errors='ignore', unk_token='<unk>', bos_token='<s>')

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def __init__(
    self,
    vocab_file,
    merges_file,
    errors="replace",
    unk_token="<|endoftext|>",
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
    pad_token=None,
    add_prefix_space=False,
    add_bos_token=False,
    **kwargs,
):
    """
    Initializes an instance of the CodeGenTokenizer class.

    Args:
        self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
        vocab_file (str): The path to the vocabulary file.
        merges_file (str): The path to the merges file.
        errors (str, optional): Specifies the error handling scheme. Defaults to 'replace'.
        unk_token (str, optional): The special token used to represent unknown tokens. Defaults to 'endoftext'.
        bos_token (str, optional): The special token used to represent the beginning of a sentence. Defaults to 'endoftext'.
        eos_token (str, optional): The special token used to represent the end of a sentence. Defaults to 'endoftext'.
        pad_token (str, optional): The special token used to represent padding. Defaults to None.
        add_prefix_space (bool, optional): Specifies whether a prefix space should be added. Defaults to False.
        add_bos_token (bool, optional): Specifies whether the bos_token should be added. Defaults to False.

    Returns:
        None.

    Raises:
        FileNotFoundError: If the vocab_file or merges_file could not be found.
        UnicodeDecodeError: If there is an error decoding the vocab_file or merges_file.
        TypeError: If any of the special token parameters (unk_token, bos_token, eos_token, pad_token) are not strings.
        TypeError: If add_bos_token or add_prefix_space are not boolean values.
        ValueError: If the merges_file is not formatted correctly.

    Note:
        This method initializes the CodeGenTokenizer class by loading the vocabulary and merges files,
        setting the error handling scheme, and initializing various attributes.

    Example:
        ```python
        >>> tokenizer = CodeGenTokenizer('vocab.txt', 'merges.txt', errors='ignore', unk_token='<unk>', bos_token='<s>')
        ```
    """
    bos_token = AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token
    eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token
    unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
    pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
    self.add_bos_token = add_bos_token

    with open(vocab_file, encoding="utf-8") as vocab_handle:
        self.encoder = json.load(vocab_handle)
    self.decoder = {v: k for k, v in self.encoder.items()}
    self.errors = errors  # how to handle errors in decoding
    self.byte_encoder = bytes_to_unicode()
    self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
    with open(merges_file, encoding="utf-8") as merges_handle:
        bpe_merges = merges_handle.read().split("\n")[1:-1]
    bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
    self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
    self.cache = {}
    self.add_prefix_space = add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
    super().__init__(
        errors=errors,
        unk_token=unk_token,
        bos_token=bos_token,
        eos_token=eos_token,
        pad_token=pad_token,
        add_prefix_space=add_prefix_space,
        add_bos_token=add_bos_token,
        **kwargs,
    )

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.bpe(token)

This method implements the Byte Pair Encoding (BPE) algorithm to tokenize a given token.

PARAMETER	DESCRIPTION
self	An instance of the CodeGenTokenizer class. TYPE: CodeGenTokenizer
token	The token to be tokenized using the BPE algorithm. TYPE: str

RETURNS	DESCRIPTION
str	The token after applying the BPE algorithm.

This method first checks if the token is already present in the cache and returns the cached value if found. It then converts the token into a tuple of characters and generates a list of all possible pairs of characters in the token. If no pairs are found, the method returns the original token as there is no need for further processing.

The method then enters a loop where it iteratively selects the pair with the lowest rank according to the BPE ranks. If the selected pair is not present in the BPE ranks, the loop is terminated.

The method then finds the first occurrence of the selected pair in the token and replaces it with the concatenation of the pair. This process is repeated until no more occurrences of the selected pair are found in the token.

Finally, the token is converted back to a string and stored in the cache for future use before being returned as the result of the method.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def bpe(self, token):
    """
    This method implements the Byte Pair Encoding (BPE) algorithm to tokenize a given token.

    Args:
        self (CodeGenTokenizer): An instance of the CodeGenTokenizer class.
        token (str): The token to be tokenized using the BPE algorithm.

    Returns:
        str: The token after applying the BPE algorithm.

    Raises:
        None.

    This method first checks if the token is already present in the cache and returns the cached value if found.
    It then converts the token into a tuple of characters and generates a list of all possible pairs of characters in the token.
    If no pairs are found, the method returns the original token as there is no need for further processing.

    The method then enters a loop where it iteratively selects the pair with the lowest rank according to the BPE ranks.
    If the selected pair is not present in the BPE ranks, the loop is terminated.

    The method then finds the first occurrence of the selected pair in the token and replaces it with the concatenation of the pair.
    This process is repeated until no more occurrences of the selected pair are found in the token.

    Finally, the token is converted back to a string and stored in the cache for future use before being returned as the result of the method.
    """
    if token in self.cache:
        return self.cache[token]
    word = tuple(token)
    pairs = get_pairs(word)

    if not pairs:
        return token

    while True:
        bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
        if bigram not in self.bpe_ranks:
            break
        first, second = bigram
        new_word = []
        i = 0
        while i < len(word):
            try:
                j = word.index(first, i)
            except ValueError:
                new_word.extend(word[i:])
                break
            else:
                new_word.extend(word[i:j])
                i = j

            if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                new_word.append(first + second)
                i += 2
            else:
                new_word.append(word[i])
                i += 1
        new_word = tuple(new_word)
        word = new_word
        if len(word) == 1:
            break
        pairs = get_pairs(word)
    word = " ".join(word)
    self.cache[token] = word
    return word

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizer class. TYPE: CodeGenTokenizer
token_ids_0	A list of token IDs for the first input sequence. TYPE: list
token_ids_1	A list of token IDs for the second input sequence. Defaults to None. TYPE: list DEFAULT: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
    """
    Args:
        self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
        token_ids_0 (list): A list of token IDs for the first input sequence.
        token_ids_1 (list, optional): A list of token IDs for the second input sequence. Defaults to None.

    Returns:
        None.

    Raises:
        None.
    """
    if self.add_bos_token:
        bos_token_ids = [self.bos_token_id]
    else:
        bos_token_ids = []

    output = bos_token_ids + token_ids_0

    if token_ids_1 is None:
        return output

    return output + bos_token_ids + token_ids_1

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    text = "".join(tokens)
    text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
    return text

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.decode(token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=None, truncate_before_pattern=None, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids)).

PARAMETER	DESCRIPTION
token_ids	List of tokenized input ids. Can be obtained using the __call__ method. TYPE: `Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`
skip_special_tokens	Whether or not to remove special tokens in the decoding. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
clean_up_tokenization_spaces	Whether or not to clean up the tokenization spaces. If None, will default to self.clean_up_tokenization_spaces (available in the tokenizer_config). TYPE: `bool`, *optional* DEFAULT: None
truncate_before_pattern	A list of regular expression strings that will be used to truncate the returned string. This can be used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning of a new line). An example pattern could be ["^#", re.escape("<|endoftext|>"), "^'''", "\n\n\n"]. TYPE: `List[str]`, *optional*, defaults to `None` DEFAULT: None
kwargs	Will be passed to the underlying model specific decode method. TYPE: additional keyword arguments, *optional* DEFAULT: {}

RETURNS	DESCRIPTION
str	str: The decoded sentence.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def decode(
    self,
    token_ids,
    skip_special_tokens: bool = False,
    clean_up_tokenization_spaces: bool = None,
    truncate_before_pattern: Optional[List[str]] = None,
    **kwargs,
) -> str:
    r"""
    Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
    tokens and clean up tokenization spaces.
    Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

    Args:
        token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
            List of tokenized input ids. Can be obtained using the `__call__` method.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.
        clean_up_tokenization_spaces (`bool`, *optional*):
            Whether or not to clean up the tokenization spaces. If `None`, will default to
            `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
        truncate_before_pattern (`List[str]`, *optional*, defaults to `None`):
            A list of regular expression strings that will be used to truncate the returned string. This can be
            used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning
            of a new line). An example pattern could be `["^#", re.escape("<|endoftext|>"), "^'''", "\n\n\n"]`.
        kwargs (additional keyword arguments, *optional*):
            Will be passed to the underlying model specific decode method.

    Returns:
        `str`: The decoded sentence.
    """
    decoded_text = super()._decode(
        token_ids=token_ids,
        skip_special_tokens=skip_special_tokens,
        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
        **kwargs,
    )

    if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
        decoded_text = self.truncate(decoded_text, truncate_before_pattern)

    return decoded_text

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.get_vocab()

Returns a dictionary containing the vocabulary of the CodeGenTokenizer instance.

PARAMETER	DESCRIPTION
self	The CodeGenTokenizer instance. TYPE: CodeGenTokenizer

RETURNS	DESCRIPTION
dict	A dictionary containing the vocabulary of the CodeGenTokenizer. The keys are tokens from the encoder and added_tokens_encoder, and the values are their corresponding indices.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def get_vocab(self):
    """
    Returns a dictionary containing the vocabulary of the CodeGenTokenizer instance.

    Args:
        self (CodeGenTokenizer): The CodeGenTokenizer instance.

    Returns:
        dict: A dictionary containing the vocabulary of the CodeGenTokenizer. The keys are tokens
            from the encoder and added_tokens_encoder, and the values are their corresponding indices.

    Raises:
        None.
    """
    return dict(self.encoder, **self.added_tokens_encoder)

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.prepare_for_tokenization(text, is_split_into_words=False, **kwargs)

Prepare the text for tokenization by adding prefix space if required.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizer class. TYPE: CodeGenTokenizer
text	The input text to be prepared for tokenization. TYPE: str
is_split_into_words	A flag indicating if the text is already split into words. Default is False. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
str	The text prepared for tokenization with potential prefix space added.

RAISES	DESCRIPTION
KeyError	If 'add_prefix_space' is not found in kwargs.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
    """
    Prepare the text for tokenization by adding prefix space if required.

    Args:
        self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
        text (str): The input text to be prepared for tokenization.
        is_split_into_words (bool): A flag indicating if the text is already split into words. Default is False.

    Returns:
        str: The text prepared for tokenization with potential prefix space added.

    Raises:
        KeyError: If 'add_prefix_space' is not found in kwargs.
    """
    add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
    if is_split_into_words or add_prefix_space:
        text = " " + text
    return (text, kwargs)

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.save_vocabulary(save_directory, filename_prefix=None)

Save the generated vocabulary files to the specified directory.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizer class. TYPE: CodeGenTokenizer
save_directory	The directory path where the vocabulary files will be saved. TYPE: str
filename_prefix	An optional prefix to be added to the filenames. Default is None. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the paths to the saved vocabulary and merge files.

RAISES	DESCRIPTION
ValueError	If the provided save_directory is not a valid directory path.
IOError	If there are issues with writing the vocabulary or merge files to the specified directory.
RuntimeError	If the BPE merge indices are not consecutive, indicating potential corruption in the tokenizer.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the generated vocabulary files to the specified directory.

    Args:
        self (CodeGenTokenizer): The instance of the CodeGenTokenizer class.
        save_directory (str): The directory path where the vocabulary files will be saved.
        filename_prefix (Optional[str]): An optional prefix to be added to the filenames. Default is None.

    Returns:
        Tuple[str]: A tuple containing the paths to the saved vocabulary and merge files.

    Raises:
        ValueError: If the provided save_directory is not a valid directory path.
        IOError: If there are issues with writing the vocabulary or merge files to the specified directory.
        RuntimeError: If the BPE merge indices are not consecutive, indicating potential corruption in the tokenizer.
    """
    if not os.path.isdir(save_directory):
        logger.error(f"Vocabulary path ({save_directory}) should be a directory")
        return
    vocab_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
    )
    merge_file = os.path.join(
        save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
    )

    with open(vocab_file, "w", encoding="utf-8") as f:
        f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

    index = 0
    with open(merge_file, "w", encoding="utf-8") as writer:
        writer.write("#version: 0.2\n")
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
            if index != token_index:
                logger.warning(
                    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                    " Please check that the tokenizer is not corrupted!"
                )
                index = token_index
            writer.write(" ".join(bpe_tokens) + "\n")
            index += 1

    return vocab_file, merge_file

mindnlp.transformers.models.codegen.tokenization_codegen.CodeGenTokenizer.truncate(completion, truncate_before_pattern)

This method 'truncate' is defined in the 'CodeGenTokenizer' class and is used to truncate a given completion based on specified patterns.

PARAMETER	DESCRIPTION
self	The instance of the class.
completion	The completion string to be truncated. TYPE: str
truncate_before_pattern	A list of patterns to truncate the completion string before. TYPE: list of str

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen.py

def truncate(self, completion, truncate_before_pattern):
    """
    This method 'truncate' is defined in the 'CodeGenTokenizer' class and is used to truncate a given completion based on specified patterns.

    Args:
        self: The instance of the class.
        completion (str): The completion string to be truncated.
        truncate_before_pattern (list of str): A list of patterns to truncate the completion string before.

    Returns:
        None.

    Raises:
        None.
    """
    def find_re(string, pattern, start_pos):
        m = pattern.search(string, start_pos)
        return m.start() if m else -1

    terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]

    prints = list(re.finditer("^print", completion, re.MULTILINE))

    if len(prints) > 1:
        completion = completion[: prints[1].start()]

    defs = list(re.finditer("^def", completion, re.MULTILINE))

    if len(defs) > 1:
        completion = completion[: defs[1].start()]

    start_pos = 0

    terminals_pos = [
        pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
    ]

    if len(terminals_pos) > 0:
        return completion[: min(terminals_pos)]
    return completion

mindnlp.transformers.models.codegen.tokenization_codegen_fast.CodeGenTokenizerFast

Bases: PreTrainedTokenizerFast

Construct a "fast" CodeGen tokenizer (backed by HuggingFace's tokenizers library). Based on byte-level Byte-Pair-Encoding.

This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will be encoded differently whether it is at the beginning of the sentence (without space) or not:

Example

>>> from transformers import CodeGenTokenizerFast
...
>>> tokenizer = CodeGenTokenizerFast.from_pretrained("Salesforce/codegen-350M-mono")
>>> tokenizer("Hello world")["input_ids"]
[15496, 995]
>>> tokenizer(" Hello world")["input_ids"]
[18435, 995]

You can get around that behavior by passing add_prefix_space=True when instantiating this tokenizer, but since the model was not pretrained this way, it might yield a decrease in performance.

When used with is_split_into_words=True, this tokenizer needs to be instantiated with add_prefix_space=True.

This tokenizer inherits from [PreTrainedTokenizerFast] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`, *optional* DEFAULT: None
merges_file	Path to the merges file. TYPE: `str`, *optional* DEFAULT: None
tokenizer_file	Path to tokenizers file (generally has a .json extension) that contains everything needed to load the tokenizer. TYPE: `str`, *optional* DEFAULT: None
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
bos_token	The beginning of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
eos_token	The end of sequence token. TYPE: `str`, *optional*, defaults to `"<|endoftext|>"` DEFAULT: '<|endoftext|>'
add_prefix_space	Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. (CodeGen tokenizer detect beginning of words by the preceding space). TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

Source code in mindnlp/transformers/models/codegen/tokenization_codegen_fast.py

class CodeGenTokenizerFast(PreTrainedTokenizerFast):
    """
    Construct a "fast" CodeGen tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
    Byte-Pair-Encoding.

    This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will
    be encoded differently whether it is at the beginning of the sentence (without space) or not:

    Example:
        ```python
        >>> from transformers import CodeGenTokenizerFast
        ...
        >>> tokenizer = CodeGenTokenizerFast.from_pretrained("Salesforce/codegen-350M-mono")
        >>> tokenizer("Hello world")["input_ids"]
        [15496, 995]
        >>> tokenizer(" Hello world")["input_ids"]
        [18435, 995]
        ```

    You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer, but since
    the model was not pretrained this way, it might yield a decrease in performance.

    <Tip>

    When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.

    </Tip>

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        merges_file (`str`, *optional*):
            Path to the merges file.
        tokenizer_file (`str`, *optional*):
            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
            contains everything needed to load the tokenizer.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        add_prefix_space (`bool`, *optional*, defaults to `False`):
            Whether or not to add an initial space to the input. This allows to treat the leading word just as any
            other word. (CodeGen tokenizer detect beginning of words by the preceding space).
    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = CodeGenTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        unk_token="<|endoftext|>",
        bos_token="<|endoftext|>",
        eos_token="<|endoftext|>",
        add_prefix_space=False,
        **kwargs,
    ):
        """
        Initializes an instance of the CodeGenTokenizerFast class.

        Args:
            vocab_file (str): The path to the vocabulary file. Default is None.
            merges_file (str): The path to the merges file. Default is None.
            tokenizer_file (str): The path to the tokenizer file. Default is None.
            unk_token (str): The unknown token to be used. Default is 'endoftext'.
            bos_token (str): The beginning of sequence token. Default is 'endoftext'.
            eos_token (str): The end of sequence token. Default is 'endoftext'.
            add_prefix_space (bool): Whether to add prefix space. Default is False.
            **kwargs: Additional keyword arguments.

        Returns:
            None

        Raises:
            ValueError: If attempting to add a BOS token using the fast tokenizer. Suggests using the slow tokenizer instead.
            JSONDecodeError: If the pre_tokenizer state cannot be decoded from JSON.
            AttributeError: If the pre_tokenizer class cannot be found.
        """
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            unk_token=unk_token,
            bos_token=bos_token,
            eos_token=eos_token,
            add_prefix_space=add_prefix_space,
            **kwargs,
        )

        if kwargs.pop("add_bos_token", False):
            model_id = kwargs.pop("name_or_path", "")
            raise ValueError(
                "Currenty GPT2's fast tokenizer does NOT support adding a BOS token. "
                "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
                f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
                f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
                "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
                " so that the fast tokenizer works correctly."
            )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
        """
        This method '_batch_encode_plus' in the class 'CodeGenTokenizerFast' encodes a batch of inputs into tokenized and encoded representations.

        Args:
            *args: Variable length positional arguments.
            **kwargs:
                Variable length keyword arguments.

                - is_split_into_words (bool, optional):
                Specifies if the input is already split into words. Defaults to False.

        Returns:
            BatchEncoding: A dictionary-like object containing the tokenized and encoded representations of the input batch.

        Raises:
            AssertionError: If the 'add_prefix_space' attribute is not set to True and the 'is_split_into_words' argument is True.
                In such cases, the method requires the instantiation of 'CodeGenTokenizerFast'
                with 'add_prefix_space=True' for using it with pretokenized inputs.
        """
        is_split_into_words = kwargs.get("is_split_into_words", False)
        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*args, **kwargs)

    def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
        """
        Encodes the input data into a batch encoding using the CodeGenTokenizerFast.

        Args:
            self: An instance of the CodeGenTokenizerFast class.

        Returns:
            A BatchEncoding object containing the encoded input data.

        Raises:
            AssertionError: If the 'is_split_into_words' keyword argument is set to True and the CodeGenTokenizerFast instance
                            was not instantiated with 'add_prefix_space=True'. This is necessary to use pretokenized inputs.

        """
        is_split_into_words = kwargs.get("is_split_into_words", False)

        assert self.add_prefix_space or not is_split_into_words, (
            f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*args, **kwargs)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary files generated by the tokenizer model.

        Args:
            self (CodeGenTokenizerFast): The instance of the CodeGenTokenizerFast class.
            save_directory (str): The directory path where the vocabulary files will be saved.
            filename_prefix (Optional[str]): An optional prefix to be added to the filename of the saved vocabulary files.
                Defaults to None if not provided.

        Returns:
            Tuple[str]: A tuple containing the filenames of the saved vocabulary files.

        Raises:
            SpecificException: Describes when a specific exception might be raised during the save operation.
            AnotherException: Describes when another type of exception might be raised during the save operation.
            AnyOtherException: Describes any other exception that the function may raise.
        """
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    def decode(
        self,
        token_ids: Union[int, List[int], "np.ndarray", "mindspore.Tensor"],
        skip_special_tokens: bool = False,
        clean_up_tokenization_spaces: bool = None,
        truncate_before_pattern: Optional[List[str]] = None,
        **kwargs,
    ) -> str:
        """
        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens
        and clean up tokenization spaces.
        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

        Args:
            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
                List of tokenized input ids. Can be obtained using the `__call__` method.
            skip_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not to remove special tokens in the decoding.
            clean_up_tokenization_spaces (`bool`, *optional*):
                Whether or not to clean up the tokenization spaces. If `None`, will default to
                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
            truncate_before_pattern (`List[str]`, *optional*, defaults to `None`):
                A list of regular expression strings that will be used to truncate the returned string. This can be
                used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning
                of a new line).
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the underlying model specific decode method.

        Returns:
            `str`: The decoded sentence.
        """
        decoded_text = super().decode(
            token_ids=token_ids,
            skip_special_tokens=skip_special_tokens,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            **kwargs,
        )

        if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
            decoded_text = self.truncate(decoded_text, truncate_before_pattern)

        return decoded_text

    def truncate(self, completion, truncate_before_pattern):
        """
        Truncate the completion string before a given pattern using a list of truncate_before_patterns.

        Args:
            self (CodeGenTokenizerFast): An instance of the CodeGenTokenizerFast class.
            completion (str): The completion string to be truncated.
            truncate_before_pattern (list): A list of patterns to truncate the completion string before.
                Each pattern is compiled using the re.compile() method with the re.MULTILINE flag.

        Returns:
            None.

        Raises:
            None.
        """
        def find_re(string, pattern, start_pos):
            m = pattern.search(string, start_pos)
            return m.start() if m else -1

        terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]

        prints = list(re.finditer("^print", completion, re.MULTILINE))

        if len(prints) > 1:
            completion = completion[: prints[1].start()]

        defs = list(re.finditer("^def", completion, re.MULTILINE))

        if len(defs) > 1:
            completion = completion[: defs[1].start()]

        start_pos = 0

        terminals_pos = [
            pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
        ]

        if len(terminals_pos) > 0:
            return completion[: min(terminals_pos)]
        return completion

mindnlp.transformers.models.codegen.tokenization_codegen_fast.CodeGenTokenizerFast.__init__(vocab_file=None, merges_file=None, tokenizer_file=None, unk_token='<|endoftext|>', bos_token='<|endoftext|>', eos_token='<|endoftext|>', add_prefix_space=False, **kwargs)

Initializes an instance of the CodeGenTokenizerFast class.

PARAMETER	DESCRIPTION
vocab_file	The path to the vocabulary file. Default is None. TYPE: str DEFAULT: None
merges_file	The path to the merges file. Default is None. TYPE: str DEFAULT: None
tokenizer_file	The path to the tokenizer file. Default is None. TYPE: str DEFAULT: None
unk_token	The unknown token to be used. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
bos_token	The beginning of sequence token. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
eos_token	The end of sequence token. Default is 'endoftext'. TYPE: str DEFAULT: '<|endoftext|>'
add_prefix_space	Whether to add prefix space. Default is False. TYPE: bool DEFAULT: False
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
ValueError	If attempting to add a BOS token using the fast tokenizer. Suggests using the slow tokenizer instead.
JSONDecodeError	If the pre_tokenizer state cannot be decoded from JSON.
AttributeError	If the pre_tokenizer class cannot be found.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen_fast.py

def __init__(
    self,
    vocab_file=None,
    merges_file=None,
    tokenizer_file=None,
    unk_token="<|endoftext|>",
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
    add_prefix_space=False,
    **kwargs,
):
    """
    Initializes an instance of the CodeGenTokenizerFast class.

    Args:
        vocab_file (str): The path to the vocabulary file. Default is None.
        merges_file (str): The path to the merges file. Default is None.
        tokenizer_file (str): The path to the tokenizer file. Default is None.
        unk_token (str): The unknown token to be used. Default is 'endoftext'.
        bos_token (str): The beginning of sequence token. Default is 'endoftext'.
        eos_token (str): The end of sequence token. Default is 'endoftext'.
        add_prefix_space (bool): Whether to add prefix space. Default is False.
        **kwargs: Additional keyword arguments.

    Returns:
        None

    Raises:
        ValueError: If attempting to add a BOS token using the fast tokenizer. Suggests using the slow tokenizer instead.
        JSONDecodeError: If the pre_tokenizer state cannot be decoded from JSON.
        AttributeError: If the pre_tokenizer class cannot be found.
    """
    super().__init__(
        vocab_file,
        merges_file,
        tokenizer_file=tokenizer_file,
        unk_token=unk_token,
        bos_token=bos_token,
        eos_token=eos_token,
        add_prefix_space=add_prefix_space,
        **kwargs,
    )

    if kwargs.pop("add_bos_token", False):
        model_id = kwargs.pop("name_or_path", "")
        raise ValueError(
            "Currenty GPT2's fast tokenizer does NOT support adding a BOS token. "
            "Instead you should use GPT2's slow tokenizer class `CodeGenTokenizer` as follows: \n"
            f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
            f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
            "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
            " so that the fast tokenizer works correctly."
        )

    pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
    if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
        pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
        pre_tok_state["add_prefix_space"] = add_prefix_space
        self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

    self.add_prefix_space = add_prefix_space

mindnlp.transformers.models.codegen.tokenization_codegen_fast.CodeGenTokenizerFast.decode(token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=None, truncate_before_pattern=None, **kwargs)

Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids)).

PARAMETER	DESCRIPTION
token_ids	List of tokenized input ids. Can be obtained using the __call__ method. TYPE: `Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`
skip_special_tokens	Whether or not to remove special tokens in the decoding. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
clean_up_tokenization_spaces	Whether or not to clean up the tokenization spaces. If None, will default to self.clean_up_tokenization_spaces (available in the tokenizer_config). TYPE: `bool`, *optional* DEFAULT: None
truncate_before_pattern	A list of regular expression strings that will be used to truncate the returned string. This can be used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning of a new line). TYPE: `List[str]`, *optional*, defaults to `None` DEFAULT: None
kwargs	Will be passed to the underlying model specific decode method. TYPE: additional keyword arguments, *optional* DEFAULT: {}

RETURNS	DESCRIPTION
str	str: The decoded sentence.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen_fast.py

def decode(
    self,
    token_ids: Union[int, List[int], "np.ndarray", "mindspore.Tensor"],
    skip_special_tokens: bool = False,
    clean_up_tokenization_spaces: bool = None,
    truncate_before_pattern: Optional[List[str]] = None,
    **kwargs,
) -> str:
    """
    Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens
    and clean up tokenization spaces.
    Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.

    Args:
        token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
            List of tokenized input ids. Can be obtained using the `__call__` method.
        skip_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to remove special tokens in the decoding.
        clean_up_tokenization_spaces (`bool`, *optional*):
            Whether or not to clean up the tokenization spaces. If `None`, will default to
            `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
        truncate_before_pattern (`List[str]`, *optional*, defaults to `None`):
            A list of regular expression strings that will be used to truncate the returned string. This can be
            used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning
            of a new line).
        kwargs (additional keyword arguments, *optional*):
            Will be passed to the underlying model specific decode method.

    Returns:
        `str`: The decoded sentence.
    """
    decoded_text = super().decode(
        token_ids=token_ids,
        skip_special_tokens=skip_special_tokens,
        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
        **kwargs,
    )

    if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
        decoded_text = self.truncate(decoded_text, truncate_before_pattern)

    return decoded_text

mindnlp.transformers.models.codegen.tokenization_codegen_fast.CodeGenTokenizerFast.save_vocabulary(save_directory, filename_prefix=None)

Save the vocabulary files generated by the tokenizer model.

PARAMETER	DESCRIPTION
self	The instance of the CodeGenTokenizerFast class. TYPE: CodeGenTokenizerFast
save_directory	The directory path where the vocabulary files will be saved. TYPE: str
filename_prefix	An optional prefix to be added to the filename of the saved vocabulary files. Defaults to None if not provided. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
Tuple[str]	Tuple[str]: A tuple containing the filenames of the saved vocabulary files.

RAISES	DESCRIPTION
SpecificException	Describes when a specific exception might be raised during the save operation.
AnotherException	Describes when another type of exception might be raised during the save operation.
AnyOtherException	Describes any other exception that the function may raise.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen_fast.py

def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
    """
    Save the vocabulary files generated by the tokenizer model.

    Args:
        self (CodeGenTokenizerFast): The instance of the CodeGenTokenizerFast class.
        save_directory (str): The directory path where the vocabulary files will be saved.
        filename_prefix (Optional[str]): An optional prefix to be added to the filename of the saved vocabulary files.
            Defaults to None if not provided.

    Returns:
        Tuple[str]: A tuple containing the filenames of the saved vocabulary files.

    Raises:
        SpecificException: Describes when a specific exception might be raised during the save operation.
        AnotherException: Describes when another type of exception might be raised during the save operation.
        AnyOtherException: Describes any other exception that the function may raise.
    """
    files = self._tokenizer.model.save(save_directory, name=filename_prefix)
    return tuple(files)

mindnlp.transformers.models.codegen.tokenization_codegen_fast.CodeGenTokenizerFast.truncate(completion, truncate_before_pattern)

Truncate the completion string before a given pattern using a list of truncate_before_patterns.

PARAMETER	DESCRIPTION
self	An instance of the CodeGenTokenizerFast class. TYPE: CodeGenTokenizerFast
completion	The completion string to be truncated. TYPE: str
truncate_before_pattern	A list of patterns to truncate the completion string before. Each pattern is compiled using the re.compile() method with the re.MULTILINE flag. TYPE: list

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/codegen/tokenization_codegen_fast.py

def truncate(self, completion, truncate_before_pattern):
    """
    Truncate the completion string before a given pattern using a list of truncate_before_patterns.

    Args:
        self (CodeGenTokenizerFast): An instance of the CodeGenTokenizerFast class.
        completion (str): The completion string to be truncated.
        truncate_before_pattern (list): A list of patterns to truncate the completion string before.
            Each pattern is compiled using the re.compile() method with the re.MULTILINE flag.

    Returns:
        None.

    Raises:
        None.
    """
    def find_re(string, pattern, start_pos):
        m = pattern.search(string, start_pos)
        return m.start() if m else -1

    terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]

    prints = list(re.finditer("^print", completion, re.MULTILINE))

    if len(prints) > 1:
        completion = completion[: prints[1].start()]

    defs = list(re.finditer("^def", completion, re.MULTILINE))

    if len(defs) > 1:
        completion = completion[: defs[1].start()]

    start_pos = 0

    terminals_pos = [
        pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
    ]

    if len(terminals_pos) > 0:
        return completion[: min(terminals_pos)]
    return completion