llama

mindnlp.transformers.models.llama.modeling_llama

MindSpore LLaMA model.

mindnlp.transformers.models.llama.modeling_llama.LlamaAttention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""
    def __init__(self, config: LlamaConfig):
        """
        Initializes an instance of the LlamaAttention class.

        Args:
            self: The instance of the LlamaAttention class.
            config (LlamaConfig):
                The configuration object that holds various parameters for the attention mechanism.

                - config.attention_dropout (float): The dropout rate for attention weights.
                - config.hidden_size (int): The size of the hidden state.
                - config.num_attention_heads (int): The number of attention heads.
                - config.num_key_value_heads (int): The number of key-value attention heads.
                - config.max_position_embeddings (int): The maximum number of position embeddings.
                - config.rope_theta (float): The rope theta value.
                - config.attention_bias (bool): Specifies whether to use bias in attention projections.

        Returns:
            None.

        Raises:
            ValueError: If the hidden_size is not divisible by num_heads.

        Note:
            This method initializes various attributes of the LlamaAttention object, such as attention_dropout, hidden_size,
            num_heads, head_dim, num_key_value_heads, num_key_value_groups, max_position_embeddings, rope_theta, and is_causal.
            It also initializes the projection layers q_proj, k_proj, v_proj, and o_proj.
            Additionally, it initializes the rope (a positional encoding used in the attention mechanism) using _init_rope method.
        """
        super().__init__()
        self.config = config
        self.attention_dropout = config.attention_dropout
        self.hidden_size = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.hidden_size // self.num_heads
        self.num_key_value_heads = config.num_key_value_heads
        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
        self.max_position_embeddings = config.max_position_embeddings
        self.rope_theta = config.rope_theta
        self.is_causal = True

        if (self.head_dim * self.num_heads) != self.hidden_size:
            raise ValueError(
                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
                f" and `num_heads`: {self.num_heads})."
            )

        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias)
        self._init_rope()

    def _init_rope(self):
        """
        Initializes the Rotary Positional Encoding (RoPE) based on the provided configuration.

        Args:
            self (LlamaAttention): The instance of the LlamaAttention class.

        Returns:
            None.

        Raises:
            ValueError: If the 'type' of RoPE scaling provided in the configuration is not recognized or supported.
        """
        if self.config.rope_scaling is None:
            self.rotary_emb = LlamaRotaryEmbedding(
                self.head_dim,
                max_position_embeddings=self.max_position_embeddings,
                base=self.rope_theta,
            )
        else:
            scaling_type = self.config.rope_scaling["type"]
            scaling_factor = self.config.rope_scaling["factor"]
            if scaling_type == "linear":
                self.rotary_emb = LlamaLinearScalingRotaryEmbedding(
                    self.head_dim,
                    max_position_embeddings=self.max_position_embeddings,
                    scaling_factor=scaling_factor,
                    base=self.rope_theta,
                )
            elif scaling_type == "dynamic":
                self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
                    self.head_dim,
                    max_position_embeddings=self.max_position_embeddings,
                    scaling_factor=scaling_factor,
                    base=self.rope_theta,
                )
            else:
                raise ValueError(f"Unknown RoPE scaling type {scaling_type}")

    def _shape(self, tensor: mindspore.Tensor, seq_len: int, bsz: int):
        """
        Reshapes the input tensor according to the specified dimensions for the LlamaAttention class.

        Args:
            self (LlamaAttention): The instance of the LlamaAttention class.
            tensor (mindspore.Tensor): The input tensor to be reshaped.
            seq_len (int): The length of the sequence.
            bsz (int): The batch size of the tensor.

        Returns:
            None

        Raises:
            None
        """
        return ops.transpose(ops.view(tensor, bsz, seq_len, self.num_heads, self.head_dim), 1, 2)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: bool = False,
        use_cache: bool = False,
        **kwargs,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        """
        This method forwards the LlamaAttention layer.

        Args:
            self: The instance of the LlamaAttention class.
            hidden_states (mindspore.Tensor): The input hidden states of shape (batch_size, sequence_length, hidden_size).
            attention_mask (Optional[mindspore.Tensor]): An optional tensor of shape
                (batch_size, 1, sequence_length, sequence_length) representing the attention mask.
            position_ids (Optional[mindspore.Tensor]): An optional tensor of shape
                (batch_size, sequence_length) representing the position ids.
            past_key_value (Optional[Tuple[mindspore.Tensor]]): An optional tuple containing the past key and value states.
            output_attentions (bool): A flag indicating whether to output attention weights.
            use_cache (bool): A flag indicating whether to use cache for past key-value states.

        Returns:
            Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]: A tuple containing
                the attention output tensor of shape (batch_size, sequence_length, hidden_size),
                optional attention weights tensor, and optional updated past key-value states tuple.

        Raises:
            ValueError: If the shape of attention weights or attention mask is not as expected.
        """
        bsz, q_len, _ = hidden_states.shape

        if self.config.pretraining_tp > 1:
            key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp
            query_slices = self.q_proj.weight.split(
                (self.num_heads * self.head_dim) // self.config.pretraining_tp, axis=0
            )
            key_slices = self.k_proj.weight.split(key_value_slicing, axis=0)
            value_slices = self.v_proj.weight.split(key_value_slicing, axis=0)

            query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)]
            query_states = ops.cat(query_states, dim=-1)

            key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)]
            key_states = ops.cat(key_states, dim=-1)

            value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)]
            value_states = ops.cat(value_states, dim=-1)

        else:
            query_states = self.q_proj(hidden_states)
            key_states = self.k_proj(hidden_states)
            value_states = self.v_proj(hidden_states)

        query_states = ops.transpose(ops.view(query_states, bsz, q_len, self.num_heads, self.head_dim), 1, 2)
        key_states = ops.transpose(ops.view(key_states, bsz, q_len, self.num_key_value_heads, self.head_dim), 1, 2)
        value_states = ops.transpose(ops.view(value_states, bsz, q_len, self.num_key_value_heads, self.head_dim), 1, 2)

        kv_seq_len = key_states.shape[-2]
        if past_key_value is not None:
            kv_seq_len += past_key_value[0].shape[-2]
        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)

        if past_key_value is not None:
            # reuse k, v, self_attention
            key_states = ops.cat([past_key_value[0], key_states], dim=2)
            value_states = ops.cat([past_key_value[1], value_states], dim=2)

        past_key_value = (key_states, value_states) if use_cache else None

        key_states = repeat_kv(key_states, self.num_key_value_groups)
        value_states = repeat_kv(value_states, self.num_key_value_groups)

        attn_weights = ops.div(ops.matmul(query_states, ops.transpose(key_states, 2, 3)), math.sqrt(self.head_dim))

        if attn_weights.shape != (bsz, self.num_heads, q_len, kv_seq_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
                f" {attn_weights.shape}"
            )

        if attention_mask is not None:
            if attention_mask.shape != (bsz, 1, q_len, kv_seq_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.shape}"
                )
            attn_weights = attn_weights + attention_mask

        # upcast attention to fp32
        attn_weights = ops.softmax(attn_weights, dim=-1, dtype=mindspore.float32).to(query_states.dtype)
        attn_weights = F.dropout(attn_weights, p=self.attention_dropout, training=self.training)
        attn_output = ops.matmul(attn_weights, value_states)

        if attn_output.shape != (bsz, self.num_heads, q_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
                f" {attn_output.shape}"
            )

        attn_output = ops.transpose(attn_output, 1, 2)

        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)

        if self.config.pretraining_tp > 1:
            attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, axis=2)
            o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, axis=1)
            attn_output = sum(F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp))
        else:
            attn_output = self.o_proj(attn_output)

        if not output_attentions:
            attn_weights = None

        return attn_output, attn_weights, past_key_value

mindnlp.transformers.models.llama.modeling_llama.LlamaAttention.__init__(config)

Initializes an instance of the LlamaAttention class.

PARAMETER	DESCRIPTION
self	The instance of the LlamaAttention class.
config	The configuration object that holds various parameters for the attention mechanism. config.attention_dropout (float): The dropout rate for attention weights. config.hidden_size (int): The size of the hidden state. config.num_attention_heads (int): The number of attention heads. config.num_key_value_heads (int): The number of key-value attention heads. config.max_position_embeddings (int): The maximum number of position embeddings. config.rope_theta (float): The rope theta value. config.attention_bias (bool): Specifies whether to use bias in attention projections. TYPE: LlamaConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the hidden_size is not divisible by num_heads.

Note

This method initializes various attributes of the LlamaAttention object, such as attention_dropout, hidden_size, num_heads, head_dim, num_key_value_heads, num_key_value_groups, max_position_embeddings, rope_theta, and is_causal. It also initializes the projection layers q_proj, k_proj, v_proj, and o_proj. Additionally, it initializes the rope (a positional encoding used in the attention mechanism) using _init_rope method.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config: LlamaConfig):
    """
    Initializes an instance of the LlamaAttention class.

    Args:
        self: The instance of the LlamaAttention class.
        config (LlamaConfig):
            The configuration object that holds various parameters for the attention mechanism.

            - config.attention_dropout (float): The dropout rate for attention weights.
            - config.hidden_size (int): The size of the hidden state.
            - config.num_attention_heads (int): The number of attention heads.
            - config.num_key_value_heads (int): The number of key-value attention heads.
            - config.max_position_embeddings (int): The maximum number of position embeddings.
            - config.rope_theta (float): The rope theta value.
            - config.attention_bias (bool): Specifies whether to use bias in attention projections.

    Returns:
        None.

    Raises:
        ValueError: If the hidden_size is not divisible by num_heads.

    Note:
        This method initializes various attributes of the LlamaAttention object, such as attention_dropout, hidden_size,
        num_heads, head_dim, num_key_value_heads, num_key_value_groups, max_position_embeddings, rope_theta, and is_causal.
        It also initializes the projection layers q_proj, k_proj, v_proj, and o_proj.
        Additionally, it initializes the rope (a positional encoding used in the attention mechanism) using _init_rope method.
    """
    super().__init__()
    self.config = config
    self.attention_dropout = config.attention_dropout
    self.hidden_size = config.hidden_size
    self.num_heads = config.num_attention_heads
    self.head_dim = self.hidden_size // self.num_heads
    self.num_key_value_heads = config.num_key_value_heads
    self.num_key_value_groups = self.num_heads // self.num_key_value_heads
    self.max_position_embeddings = config.max_position_embeddings
    self.rope_theta = config.rope_theta
    self.is_causal = True

    if (self.head_dim * self.num_heads) != self.hidden_size:
        raise ValueError(
            f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
            f" and `num_heads`: {self.num_heads})."
        )

    self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias)
    self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
    self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias)
    self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias)
    self._init_rope()

mindnlp.transformers.models.llama.modeling_llama.LlamaAttention.forward(hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, **kwargs)

This method forwards the LlamaAttention layer.

PARAMETER	DESCRIPTION
self	The instance of the LlamaAttention class.
hidden_states	The input hidden states of shape (batch_size, sequence_length, hidden_size). TYPE: Tensor
attention_mask	An optional tensor of shape (batch_size, 1, sequence_length, sequence_length) representing the attention mask. TYPE: Optional[Tensor] DEFAULT: None
position_ids	An optional tensor of shape (batch_size, sequence_length) representing the position ids. TYPE: Optional[Tensor] DEFAULT: None
past_key_value	An optional tuple containing the past key and value states. TYPE: Optional[Tuple[Tensor]] DEFAULT: None
output_attentions	A flag indicating whether to output attention weights. TYPE: bool DEFAULT: False
use_cache	A flag indicating whether to use cache for past key-value states. TYPE: bool DEFAULT: False

RETURNS	DESCRIPTION
Tuple[Tensor, Optional[Tensor], Optional[Tuple[Tensor]]]	Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]: A tuple containing the attention output tensor of shape (batch_size, sequence_length, hidden_size), optional attention weights tensor, and optional updated past key-value states tuple.

RAISES	DESCRIPTION
ValueError	If the shape of attention weights or attention mask is not as expected.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: bool = False,
    use_cache: bool = False,
    **kwargs,
) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
    """
    This method forwards the LlamaAttention layer.

    Args:
        self: The instance of the LlamaAttention class.
        hidden_states (mindspore.Tensor): The input hidden states of shape (batch_size, sequence_length, hidden_size).
        attention_mask (Optional[mindspore.Tensor]): An optional tensor of shape
            (batch_size, 1, sequence_length, sequence_length) representing the attention mask.
        position_ids (Optional[mindspore.Tensor]): An optional tensor of shape
            (batch_size, sequence_length) representing the position ids.
        past_key_value (Optional[Tuple[mindspore.Tensor]]): An optional tuple containing the past key and value states.
        output_attentions (bool): A flag indicating whether to output attention weights.
        use_cache (bool): A flag indicating whether to use cache for past key-value states.

    Returns:
        Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]: A tuple containing
            the attention output tensor of shape (batch_size, sequence_length, hidden_size),
            optional attention weights tensor, and optional updated past key-value states tuple.

    Raises:
        ValueError: If the shape of attention weights or attention mask is not as expected.
    """
    bsz, q_len, _ = hidden_states.shape

    if self.config.pretraining_tp > 1:
        key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp
        query_slices = self.q_proj.weight.split(
            (self.num_heads * self.head_dim) // self.config.pretraining_tp, axis=0
        )
        key_slices = self.k_proj.weight.split(key_value_slicing, axis=0)
        value_slices = self.v_proj.weight.split(key_value_slicing, axis=0)

        query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)]
        query_states = ops.cat(query_states, dim=-1)

        key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)]
        key_states = ops.cat(key_states, dim=-1)

        value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)]
        value_states = ops.cat(value_states, dim=-1)

    else:
        query_states = self.q_proj(hidden_states)
        key_states = self.k_proj(hidden_states)
        value_states = self.v_proj(hidden_states)

    query_states = ops.transpose(ops.view(query_states, bsz, q_len, self.num_heads, self.head_dim), 1, 2)
    key_states = ops.transpose(ops.view(key_states, bsz, q_len, self.num_key_value_heads, self.head_dim), 1, 2)
    value_states = ops.transpose(ops.view(value_states, bsz, q_len, self.num_key_value_heads, self.head_dim), 1, 2)

    kv_seq_len = key_states.shape[-2]
    if past_key_value is not None:
        kv_seq_len += past_key_value[0].shape[-2]
    cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
    query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)

    if past_key_value is not None:
        # reuse k, v, self_attention
        key_states = ops.cat([past_key_value[0], key_states], dim=2)
        value_states = ops.cat([past_key_value[1], value_states], dim=2)

    past_key_value = (key_states, value_states) if use_cache else None

    key_states = repeat_kv(key_states, self.num_key_value_groups)
    value_states = repeat_kv(value_states, self.num_key_value_groups)

    attn_weights = ops.div(ops.matmul(query_states, ops.transpose(key_states, 2, 3)), math.sqrt(self.head_dim))

    if attn_weights.shape != (bsz, self.num_heads, q_len, kv_seq_len):
        raise ValueError(
            f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
            f" {attn_weights.shape}"
        )

    if attention_mask is not None:
        if attention_mask.shape != (bsz, 1, q_len, kv_seq_len):
            raise ValueError(
                f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.shape}"
            )
        attn_weights = attn_weights + attention_mask

    # upcast attention to fp32
    attn_weights = ops.softmax(attn_weights, dim=-1, dtype=mindspore.float32).to(query_states.dtype)
    attn_weights = F.dropout(attn_weights, p=self.attention_dropout, training=self.training)
    attn_output = ops.matmul(attn_weights, value_states)

    if attn_output.shape != (bsz, self.num_heads, q_len, self.head_dim):
        raise ValueError(
            f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
            f" {attn_output.shape}"
        )

    attn_output = ops.transpose(attn_output, 1, 2)

    attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)

    if self.config.pretraining_tp > 1:
        attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, axis=2)
        o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, axis=1)
        attn_output = sum(F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp))
    else:
        attn_output = self.o_proj(attn_output)

    if not output_attentions:
        attn_weights = None

    return attn_output, attn_weights, past_key_value

mindnlp.transformers.models.llama.modeling_llama.LlamaDecoderLayer

Bases: Module

The LlamaDecoderLayer class represents a layer of the Llama decoder in the Llama model. It inherits from the nn.Module class.

ATTRIBUTE	DESCRIPTION
hidden_size	The size of the hidden layer. TYPE: int
self_attn	The attention mechanism used in the layer. TYPE: `LlamaAttention`
mlp	The multi-layer perceptron used in the layer. TYPE: `LlamaMLP`
input_layernorm	The input layer normalization module. TYPE: `LlamaRMSNorm`
post_attention_layernorm	The layer normalization module applied after the attention mechanism. TYPE: `LlamaRMSNorm`

METHOD

DESCRIPTION

forward

Applies the Llama decoder layer to the input hidden states. Args: hidden_states (mindspore.Tensor): The input to the layer of shape (batch, seq_len, embed_dim). attention_mask (mindspore.Tensor, optional): The attention mask. Its shape depends on the attention mechanism used. For flash attention, it has a shape of (batch_size, sequence_length), and for default attention, it has a shape of (batch_size, 1, query_sequence_length, key_sequence_length). position_ids (mindspore.Tensor, optional): The position ids tensor. past_key_value (Tuple[mindspore.Tensor], optional): The cached past key and value projection states. output_attentions (bool, optional): Whether or not to return the attention tensors of all attention layers. See the attentions under the returned tensors for more detail. use_cache (bool, optional): If set to True, the past_key_values key value states are returned and can be used to speed up decoding. See past_key_values for more information. kwargs: Additional keyword arguments. Returns: Tuple[mindspore.Tensor, Optional[Tuple[mindspore.Tensor, mindspore.Tensor]]]: The output tensor of shape (batch, seq_len, embed_dim). If output_attentions is True, the tuple also includes the attention weights tensor. If use_cache is True, the tuple also includes the present key and value projection states.

Note

The LlamaDecoderLayer class assumes that the LlamaConfig instance is already defined and passed as an argument to the forwardor.

Example

>>> # Create a LlamaDecoderLayer instance
>>> config = LlamaConfig(hidden_size=512)
>>> decoder_layer = LlamaDecoderLayer(config)
...
>>> # Apply the Llama decoder layer to the hidden states
>>> hidden_states = ...
>>> attention_mask = ...
>>> output = decoder_layer.forward(hidden_states, attention_mask)

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaDecoderLayer(nn.Module):

    """
    The `LlamaDecoderLayer` class represents a layer of the Llama decoder in the Llama model.
    It inherits from the `nn.Module` class.

    Attributes:
        hidden_size (int): The size of the hidden layer.
        self_attn (`LlamaAttention`): The attention mechanism used in the layer.
        mlp (`LlamaMLP`): The multi-layer perceptron used in the layer.
        input_layernorm (`LlamaRMSNorm`): The input layer normalization module.
        post_attention_layernorm (`LlamaRMSNorm`): The layer normalization module applied after the attention mechanism.

    Methods:
        forward:
            Applies the Llama decoder layer to the input hidden states.

            Args:

            - hidden_states (mindspore.Tensor): The input to the layer of shape `(batch, seq_len, embed_dim)`.
            - attention_mask (mindspore.Tensor, optional): The attention mask. Its shape depends on the attention
            mechanism used. For flash attention, it has a shape of `(batch_size, sequence_length)`, and
            for default attention, it has a shape of `(batch_size, 1, query_sequence_length, key_sequence_length)`.
            - position_ids (mindspore.Tensor, optional): The position ids tensor.
            - past_key_value (Tuple[mindspore.Tensor], optional): The cached past key and value projection states.
            - output_attentions (bool, optional): Whether or not to return the attention tensors of all attention layers.
            See the `attentions` under the returned tensors for more detail.
            - use_cache (bool, optional): If set to True, the `past_key_values` key value states are returned and can be
            used to speed up decoding. See `past_key_values` for more information.
            - kwargs: Additional keyword arguments.

            Returns:

            - Tuple[mindspore.Tensor, Optional[Tuple[mindspore.Tensor, mindspore.Tensor]]]:
            The output tensor of shape `(batch, seq_len, embed_dim)`.
            If `output_attentions` is True, the tuple also includes
            - the attention weights tensor.
            If `use_cache` is True, the tuple also includes the present key and value projection states.

    Note:
        The `LlamaDecoderLayer` class assumes that the `LlamaConfig` instance is already defined and passed as
        an argument to the forwardor.

    Example:
        ```python
        >>> # Create a LlamaDecoderLayer instance
        >>> config = LlamaConfig(hidden_size=512)
        >>> decoder_layer = LlamaDecoderLayer(config)
        ...
        >>> # Apply the Llama decoder layer to the hidden states
        >>> hidden_states = ...
        >>> attention_mask = ...
        >>> output = decoder_layer.forward(hidden_states, attention_mask)
        ```
    """
    def __init__(self, config: LlamaConfig):
        """
        Initializes a LlamaDecoderLayer instance.

        Args:
            self (LlamaDecoderLayer): The current instance of the LlamaDecoderLayer class.
            config (LlamaConfig): An object of type LlamaConfig containing configuration parameters for the decoder layer.
                The config object must have the following attributes:

                - hidden_size (int): The size of the hidden layers.
                - rms_norm_eps (float): The epsilon value for RMS normalization.

        Returns:
            None.

        Raises:
            TypeError: If config is not an instance of LlamaConfig.
            ValueError: If config is missing any required attribute.
        """
        super().__init__()
        self.hidden_size = config.hidden_size
        self.self_attn = LlamaAttention(config=config)

        self.mlp = LlamaMLP(config)
        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = False,
        **kwargs,
    ) -> Tuple[mindspore.Tensor, Optional[Tuple[mindspore.Tensor, mindspore.Tensor]]]:
        """
        Args:
            hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(mindspore.Tensor)`, *optional*): cached past key and value projection states
        """
        residual = hidden_states

        hidden_states = self.input_layernorm(hidden_states)

        # Self Attention
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_value=past_key_value,
            output_attentions=output_attentions,
            use_cache=use_cache,
            **kwargs,
        )
        hidden_states = residual + hidden_states

        # Fully Connected
        residual = hidden_states
        hidden_states = self.post_attention_layernorm(hidden_states)
        hidden_states = self.mlp(hidden_states)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights,)

        if use_cache:
            outputs += (present_key_value,)

        return outputs

mindnlp.transformers.models.llama.modeling_llama.LlamaDecoderLayer.__init__(config)

Initializes a LlamaDecoderLayer instance.

PARAMETER	DESCRIPTION
self	The current instance of the LlamaDecoderLayer class. TYPE: LlamaDecoderLayer
config	An object of type LlamaConfig containing configuration parameters for the decoder layer. The config object must have the following attributes: hidden_size (int): The size of the hidden layers. rms_norm_eps (float): The epsilon value for RMS normalization. TYPE: LlamaConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If config is not an instance of LlamaConfig.
ValueError	If config is missing any required attribute.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config: LlamaConfig):
    """
    Initializes a LlamaDecoderLayer instance.

    Args:
        self (LlamaDecoderLayer): The current instance of the LlamaDecoderLayer class.
        config (LlamaConfig): An object of type LlamaConfig containing configuration parameters for the decoder layer.
            The config object must have the following attributes:

            - hidden_size (int): The size of the hidden layers.
            - rms_norm_eps (float): The epsilon value for RMS normalization.

    Returns:
        None.

    Raises:
        TypeError: If config is not an instance of LlamaConfig.
        ValueError: If config is missing any required attribute.
    """
    super().__init__()
    self.hidden_size = config.hidden_size
    self.self_attn = LlamaAttention(config=config)

    self.mlp = LlamaMLP(config)
    self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
    self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

mindnlp.transformers.models.llama.modeling_llama.LlamaDecoderLayer.forward(hidden_states, attention_mask=None, position_ids=None, past_key_value=None, output_attentions=False, use_cache=False, **kwargs)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch_size, sequence_length) if flash attention is used or (batch_size, 1, query_sequence_length, key_sequence_length) if default attention is used. TYPE: `mindspore.Tensor`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False
use_cache	If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values). TYPE: `bool`, *optional* DEFAULT: False
past_key_value	cached past key and value projection states TYPE: `Tuple(mindspore.Tensor)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: Optional[bool] = False,
    use_cache: Optional[bool] = False,
    **kwargs,
) -> Tuple[mindspore.Tensor, Optional[Tuple[mindspore.Tensor, mindspore.Tensor]]]:
    """
    Args:
        hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`, *optional*):
            attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
            query_sequence_length, key_sequence_length)` if default attention is used.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
            (see `past_key_values`).
        past_key_value (`Tuple(mindspore.Tensor)`, *optional*): cached past key and value projection states
    """
    residual = hidden_states

    hidden_states = self.input_layernorm(hidden_states)

    # Self Attention
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_value=past_key_value,
        output_attentions=output_attentions,
        use_cache=use_cache,
        **kwargs,
    )
    hidden_states = residual + hidden_states

    # Fully Connected
    residual = hidden_states
    hidden_states = self.post_attention_layernorm(hidden_states)
    hidden_states = self.mlp(hidden_states)
    hidden_states = residual + hidden_states

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (self_attn_weights,)

    if use_cache:
        outputs += (present_key_value,)

    return outputs

mindnlp.transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding

Bases: LlamaRotaryEmbedding

LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding):
    """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
    def __init__(self, dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0):
        """
        Initializes an instance of the LlamaDynamicNTKScalingRotaryEmbedding class.

        Args:
            self: The instance of the class.
            dim (int): The dimension of the embedding.
            max_position_embeddings (int): The maximum number of position embeddings to be considered. Default is 2048.
            base (int): The base value used in calculations. Default is 10000.
            scaling_factor (float): The scaling factor applied to the embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.scaling_factor = scaling_factor
        super().__init__(dim, max_position_embeddings, base)

    def _set_cos_sin_cache(self, seq_len, dtype):
        """
        Method to set the cosine and sine cache for dynamic NTK scaling rotary embedding in the
        LlamaDynamicNTKScalingRotaryEmbedding class.

        Args:
            self: Instance of the LlamaDynamicNTKScalingRotaryEmbedding class.
            seq_len: Integer representing the length of the sequence for which the cosine and sine cache is being set.
            dtype: Data type of the elements in the cache.

        Returns:
            None: This method updates the cosine and sine cache attributes of the instance.

        Raises:
            ValueError: If the input sequence length 'seq_len' is not a positive integer.
            ValueError: If the input data type 'dtype' is not a valid data type.
            RuntimeError: If an error occurs during the calculation of the cosine and sine cache.
        """
        self.max_seq_len_cached = seq_len

        if seq_len > self.max_position_embeddings:
            base = self.base * (
                (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
            ) ** (self.dim / (self.dim - 2))
            inv_freq = 1.0 / (base ** (ops.arange(0, self.dim, 2).float() / self.dim))
            self.inv_freq = inv_freq

        t = ops.arange(self.max_seq_len_cached, dtype=self.inv_freq.dtype)

        freqs = ops.outer(t, self.inv_freq)
        # Different from paper, but it uses a different permutation in order to obtain the same calculation
        emb = ops.cat((freqs, freqs), dim=-1)
        self.cos_cached = ops.cos(emb).to(dtype)
        self.sin_cached = ops.sin(emb).to(dtype)

mindnlp.transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding.__init__(dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0)

Initializes an instance of the LlamaDynamicNTKScalingRotaryEmbedding class.

PARAMETER	DESCRIPTION
self	The instance of the class.
dim	The dimension of the embedding. TYPE: int
max_position_embeddings	The maximum number of position embeddings to be considered. Default is 2048. TYPE: int DEFAULT: 2048
base	The base value used in calculations. Default is 10000. TYPE: int DEFAULT: 10000
scaling_factor	The scaling factor applied to the embeddings. TYPE: float DEFAULT: 1.0

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0):
    """
    Initializes an instance of the LlamaDynamicNTKScalingRotaryEmbedding class.

    Args:
        self: The instance of the class.
        dim (int): The dimension of the embedding.
        max_position_embeddings (int): The maximum number of position embeddings to be considered. Default is 2048.
        base (int): The base value used in calculations. Default is 10000.
        scaling_factor (float): The scaling factor applied to the embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.scaling_factor = scaling_factor
    super().__init__(dim, max_position_embeddings, base)

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM

Bases: LlamaPreTrainedModel

This class represents a Llama model for Causal Language Modeling (LM) tasks. It includes methods for setting and getting input and output embeddings, setting and getting the decoder, as well as methods for model forwardion and preparing inputs for generation. The class inherits from LlamaPreTrainedModel and implements the necessary functionalities for generating text based on a given prompt.

ATTRIBUTE	DESCRIPTION
model	Instance of LlamaModel used for the LM task.
vocab_size	Size of the vocabulary used in the LM task.
lm_head	Neural network layer for LM head.

METHOD	DESCRIPTION
get_input_embeddings	Retrieve the input embeddings from the model.
set_input_embeddings	Set new input embeddings for the model.
get_output_embeddings	Get the output embeddings for the LM task.
set_output_embeddings	Set new output embeddings.
set_decoder	Set a new decoder for the model.
get_decoder	Get the current decoder used in the model.
forward	forward the model for the LM task with specified inputs and return the outputs.
prepare_inputs_for_generation	Prepare input data for text generation based on past key values and attention mask.
_reorder_cache	Reorder cache elements based on beam index for efficient generation.

Example

>>> from transformers import AutoTokenizer, LlamaForCausalLM
...
>>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
>>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
...
>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")
...
>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaForCausalLM(LlamaPreTrainedModel):
    r"""
    This class represents a Llama model for Causal Language Modeling (LM) tasks.
    It includes methods for setting and getting input and output embeddings, setting and getting the decoder,
    as well as methods for model forwardion and preparing inputs for generation.
    The class inherits from LlamaPreTrainedModel and implements the necessary functionalities for generating text
    based on a given prompt.

    Attributes:
        model: Instance of LlamaModel used for the LM task.
        vocab_size: Size of the vocabulary used in the LM task.
        lm_head: Neural network layer for LM head.

    Methods:
        get_input_embeddings(): Retrieve the input embeddings from the model.
        set_input_embeddings(value): Set new input embeddings for the model.
        get_output_embeddings(): Get the output embeddings for the LM task.
        set_output_embeddings(new_embeddings): Set new output embeddings.
        set_decoder(decoder): Set a new decoder for the model.
        get_decoder(): Get the current decoder used in the model.
        forward(): forward the model for the LM task with specified inputs and return the outputs.
        prepare_inputs_for_generation(): Prepare input data for text generation based on past key values and attention mask.
        _reorder_cache(past_key_values, beam_idx): Reorder cache elements based on beam index for efficient generation.

    Example:
        ```python
        >>> from transformers import AutoTokenizer, LlamaForCausalLM
        ...
        >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
        ...
        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")
        ...
        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```
    """
    _tied_weights_keys = ["lm_head.weight"]

    def __init__(self, config):
        r"""
        Initializes an instance of the LlamaForCausalLM class.

        Args:
            self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
            config (dict): The configuration dictionary containing parameters for model initialization.
                Must include the following keys:

                - vocab_size (int): The size of the vocabulary.
                - hidden_size (int): The size of the hidden layers in the model.

        Returns:
            None.

        Raises:
            ValueError: If the 'config' dictionary is missing required keys or if the values are of incorrect types.
            TypeError: If 'config' is not a dictionary or if any of the values in the 'config'
                dictionary are of incorrect types.
            RuntimeError: If an error occurs during model initialization.
        """
        super().__init__(config)
        self.model = LlamaModel(config)
        self.vocab_size = config.vocab_size
        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Method to retrieve input embeddings from the 'LlamaForCausalLM' class model.

        Args:
            self (LlamaForCausalLM): The instance of the 'LlamaForCausalLM' class.
                This parameter is used to access the model's embed tokens for input embeddings.

        Returns:
            None: This method returns None as it directly retrieves and returns the input embeddings from the model.

        Raises:
            None.
        """
        return self.model.embed_tokens

    def set_input_embeddings(self, value):
        """
        Method: set_input_embeddings

        Description: Sets the input embeddings of the LlamaForCausalLM model.

        Args:
            self (LlamaForCausalLM):
                The instance of the LlamaForCausalLM class.

                - Type: LlamaForCausalLM
                - Purpose: Represents the current instance of the LlamaForCausalLM class.
                - Restrictions: Must be an instance of the LlamaForCausalLM class.

            value:
                The input embeddings to be set for the model.

                - Type: Any
                - Purpose: Represents the new input embeddings to be assigned to the model.
                - Restrictions: None

        Returns:
            None.

        Raises:
            None
        """
        self.model.embed_tokens = value

    def get_output_embeddings(self):
        """
        Retrieve the output embeddings from the LlamaForCausalLM model.

        Args:
            self: An instance of the LlamaForCausalLM class.

        Returns:
            lm_head: This method returns the output embeddings from the lm_head layer of the LlamaForCausalLM model.

        Raises:
            None.
        """
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        """
        Sets the output embeddings for the LlamaForCausalLM model.

        Args:
            self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
            new_embeddings (Tensor): The new embeddings to be set for the model's lm_head.

        Returns:
            None.

        Raises:
            None.

        This method allows the user to update the output embeddings of the LlamaForCausalLM model by replacing the
        current embeddings with the provided new_embeddings.
        The new_embeddings should be a tensor of the same shape and size as the current embeddings.
        This method is useful in fine-tuning the model with custom embeddings or when transferring the model to
        a different task that requires different output embeddings.
        """
        self.lm_head = new_embeddings

    def set_decoder(self, decoder):
        """
        Sets the decoder for the LlamaForCausalLM model.

        Args:
            self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
            decoder: The decoder object to be set for the model.

        Returns:
            None.

        Raises:
            None.

        This method sets the decoder object provided as an argument to the 'model' attribute of the
        LlamaForCausalLM instance.
        The 'model' attribute represents the decoder used for the causal language modeling task.
        """
        self.model = decoder

    def get_decoder(self):
        """
        This method returns the decoder model used for the LlamaForCausalLM class.

        Args:
            self: The instance of the LlamaForCausalLM class.

        Returns:
            None: This method returns the decoder model associated with the LlamaForCausalLM instance.

        Raises:
            None.
        """
        return self.model

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithPast]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:
            Union[Tuple, CausalLMOutputWithPast]

        Example:
            ```python
            >>> from transformers import AutoTokenizer, LlamaForCausalLM
            ...
            >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
            >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
            ...
            >>> prompt = "Hey, are you conscious? Can you talk to me?"
            >>> inputs = tokenizer(prompt, return_tensors="pt")
            ...
            >>> # Generate
            >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
            >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
            "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
            ```
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        hidden_states = outputs[0]
        if self.config.pretraining_tp > 1:
            lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0)
            logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)]
            logits = ops.cat(logits, dim=-1)
        else:
            logits = self.lm_head(hidden_states)
        logits = logits.float()

        loss = None
        if labels is not None:
            # Shift so that tokens < n predict n
            shift_logits = logits[..., :-1, :]
            shift_labels = labels[..., 1:]
            # Flatten the tokens
            shift_logits = shift_logits.view(-1, self.config.vocab_size)
            shift_labels = shift_labels.view(-1)
            # Enable model parallelism
            loss = F.cross_entropy(shift_logits, shift_labels)

        if not return_dict:
            output = (logits,) + outputs[1:]
            return (loss,) + output if loss is not None else output

        return CausalLMOutputWithPast(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
    ):
        """
        Method to prepare inputs for generation in the LlamaForCausalLM class.

        Args:
            self (object): The instance of the class.
            input_ids (torch.Tensor): The input tensor representing tokenized input sequence.
            past_key_values (tuple, optional): Tuple containing past key values for autoregressive generation.
                Default is None.
            attention_mask (torch.Tensor, optional): Mask tensor indicating attention areas. Default is None.
            inputs_embeds (torch.Tensor, optional): Embedding tensor for the input tokens. Default is None.
            **kwargs: Additional keyword arguments.

        Returns:
            dict: A dictionary containing the prepared model inputs including 'input_ids', 'position_ids',
                'past_key_values', 'use_cache', and 'attention_mask'.

        Raises:
            ValueError: If the input_ids shape is incorrect or if attention_mask is not provided.
            TypeError: If the position_ids are not of type torch.Tensor.
            RuntimeError: If an unexpected error occurs during position_ids calculation.
        """
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        position_ids = kwargs.get("position_ids", None)
        if attention_mask is not None and position_ids is None:
            # create position_ids on the fly for batch generation
            position_ids = ops.cumsum(attention_mask.long(), -1) - 1
            position_ids = position_ids.masked_fill(attention_mask == 0, 1)
            if past_key_values:
                position_ids = position_ids[:, -input_ids.shape[1] :]

        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
        if inputs_embeds is not None and past_key_values is None:
            model_inputs = {"inputs_embeds": inputs_embeds}
        else:
            model_inputs = {"input_ids": input_ids}

        model_inputs.update(
            {
                "position_ids": position_ids,
                "past_key_values": past_key_values,
                "use_cache": kwargs.get("use_cache"),
                "attention_mask": attention_mask,
            }
        )
        return model_inputs

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        """
        Reorders the cache of past key values for a given beam index in the LlamaForCausalLM class.

        Args:
            past_key_values (tuple): A tuple containing the cache of past key values.
                Each element in the tuple represents the cache for a particular layer.
            beam_idx (int): The index of the beam for which the cache is to be reordered.

        Returns:
            None: This method modifies the existing cache in-place.

        Raises:
            None.

        This static method reorders the cache of past key values for a specific beam index in the LlamaForCausalLM class.
        The method iterates over each layer's cache and reorders the past states based on the provided beam index.
        The reordered cache is then returned as a tuple of past key values. The original cache is modified in-place
        and no new objects are created.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__(config)

Initializes an instance of the LlamaForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the LlamaForCausalLM class. TYPE: LlamaForCausalLM
config	The configuration dictionary containing parameters for model initialization. Must include the following keys: vocab_size (int): The size of the vocabulary. hidden_size (int): The size of the hidden layers in the model. TYPE: dict

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the 'config' dictionary is missing required keys or if the values are of incorrect types.
TypeError	If 'config' is not a dictionary or if any of the values in the 'config' dictionary are of incorrect types.
RuntimeError	If an error occurs during model initialization.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config):
    r"""
    Initializes an instance of the LlamaForCausalLM class.

    Args:
        self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
        config (dict): The configuration dictionary containing parameters for model initialization.
            Must include the following keys:

            - vocab_size (int): The size of the vocabulary.
            - hidden_size (int): The size of the hidden layers in the model.

    Returns:
        None.

    Raises:
        ValueError: If the 'config' dictionary is missing required keys or if the values are of incorrect types.
        TypeError: If 'config' is not a dictionary or if any of the values in the 'config'
            dictionary are of incorrect types.
        RuntimeError: If an error occurs during model initialization.
    """
    super().__init__(config)
    self.model = LlamaModel(config)
    self.vocab_size = config.vocab_size
    self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.forward(input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the masked language modeling loss. Indices should either be in [0, ..., config.vocab_size] or -100 (see input_ids docstring). Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, CausalLMOutputWithPast]	Union[Tuple, CausalLMOutputWithPast]

Example

>>> from transformers import AutoTokenizer, LlamaForCausalLM
...
>>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
>>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
...
>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")
...
>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, CausalLMOutputWithPast]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

    Returns:
        Union[Tuple, CausalLMOutputWithPast]

    Example:
        ```python
        >>> from transformers import AutoTokenizer, LlamaForCausalLM
        ...
        >>> model = LlamaForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
        ...
        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")
        ...
        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    outputs = self.model(
        input_ids=input_ids,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    hidden_states = outputs[0]
    if self.config.pretraining_tp > 1:
        lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0)
        logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)]
        logits = ops.cat(logits, dim=-1)
    else:
        logits = self.lm_head(hidden_states)
    logits = logits.float()

    loss = None
    if labels is not None:
        # Shift so that tokens < n predict n
        shift_logits = logits[..., :-1, :]
        shift_labels = labels[..., 1:]
        # Flatten the tokens
        shift_logits = shift_logits.view(-1, self.config.vocab_size)
        shift_labels = shift_labels.view(-1)
        # Enable model parallelism
        loss = F.cross_entropy(shift_logits, shift_labels)

    if not return_dict:
        output = (logits,) + outputs[1:]
        return (loss,) + output if loss is not None else output

    return CausalLMOutputWithPast(
        loss=loss,
        logits=logits,
        past_key_values=outputs.past_key_values,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
    )

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder()

This method returns the decoder model used for the LlamaForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the LlamaForCausalLM class.

RETURNS	DESCRIPTION
None	This method returns the decoder model associated with the LlamaForCausalLM instance.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def get_decoder(self):
    """
    This method returns the decoder model used for the LlamaForCausalLM class.

    Args:
        self: The instance of the LlamaForCausalLM class.

    Returns:
        None: This method returns the decoder model associated with the LlamaForCausalLM instance.

    Raises:
        None.
    """
    return self.model

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings()

Method to retrieve input embeddings from the 'LlamaForCausalLM' class model.

PARAMETER	DESCRIPTION
self	The instance of the 'LlamaForCausalLM' class. This parameter is used to access the model's embed tokens for input embeddings. TYPE: LlamaForCausalLM

RETURNS	DESCRIPTION
None	This method returns None as it directly retrieves and returns the input embeddings from the model.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def get_input_embeddings(self):
    """
    Method to retrieve input embeddings from the 'LlamaForCausalLM' class model.

    Args:
        self (LlamaForCausalLM): The instance of the 'LlamaForCausalLM' class.
            This parameter is used to access the model's embed tokens for input embeddings.

    Returns:
        None: This method returns None as it directly retrieves and returns the input embeddings from the model.

    Raises:
        None.
    """
    return self.model.embed_tokens

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings()

Retrieve the output embeddings from the LlamaForCausalLM model.

PARAMETER	DESCRIPTION
self	An instance of the LlamaForCausalLM class.

RETURNS	DESCRIPTION
lm_head	This method returns the output embeddings from the lm_head layer of the LlamaForCausalLM model.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def get_output_embeddings(self):
    """
    Retrieve the output embeddings from the LlamaForCausalLM model.

    Args:
        self: An instance of the LlamaForCausalLM class.

    Returns:
        lm_head: This method returns the output embeddings from the lm_head layer of the LlamaForCausalLM model.

    Raises:
        None.
    """
    return self.lm_head

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs)

Method to prepare inputs for generation in the LlamaForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
input_ids	The input tensor representing tokenized input sequence. TYPE: Tensor
past_key_values	Tuple containing past key values for autoregressive generation. Default is None. TYPE: tuple DEFAULT: None
attention_mask	Mask tensor indicating attention areas. Default is None. TYPE: Tensor DEFAULT: None
inputs_embeds	Embedding tensor for the input tokens. Default is None. TYPE: Tensor DEFAULT: None
**kwargs	Additional keyword arguments. DEFAULT: {}

RETURNS	DESCRIPTION
dict	A dictionary containing the prepared model inputs including 'input_ids', 'position_ids', 'past_key_values', 'use_cache', and 'attention_mask'.

RAISES	DESCRIPTION
ValueError	If the input_ids shape is incorrect or if attention_mask is not provided.
TypeError	If the position_ids are not of type torch.Tensor.
RuntimeError	If an unexpected error occurs during position_ids calculation.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def prepare_inputs_for_generation(
    self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
):
    """
    Method to prepare inputs for generation in the LlamaForCausalLM class.

    Args:
        self (object): The instance of the class.
        input_ids (torch.Tensor): The input tensor representing tokenized input sequence.
        past_key_values (tuple, optional): Tuple containing past key values for autoregressive generation.
            Default is None.
        attention_mask (torch.Tensor, optional): Mask tensor indicating attention areas. Default is None.
        inputs_embeds (torch.Tensor, optional): Embedding tensor for the input tokens. Default is None.
        **kwargs: Additional keyword arguments.

    Returns:
        dict: A dictionary containing the prepared model inputs including 'input_ids', 'position_ids',
            'past_key_values', 'use_cache', and 'attention_mask'.

    Raises:
        ValueError: If the input_ids shape is incorrect or if attention_mask is not provided.
        TypeError: If the position_ids are not of type torch.Tensor.
        RuntimeError: If an unexpected error occurs during position_ids calculation.
    """
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    position_ids = kwargs.get("position_ids", None)
    if attention_mask is not None and position_ids is None:
        # create position_ids on the fly for batch generation
        position_ids = ops.cumsum(attention_mask.long(), -1) - 1
        position_ids = position_ids.masked_fill(attention_mask == 0, 1)
        if past_key_values:
            position_ids = position_ids[:, -input_ids.shape[1] :]

    # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
    if inputs_embeds is not None and past_key_values is None:
        model_inputs = {"inputs_embeds": inputs_embeds}
    else:
        model_inputs = {"input_ids": input_ids}

    model_inputs.update(
        {
            "position_ids": position_ids,
            "past_key_values": past_key_values,
            "use_cache": kwargs.get("use_cache"),
            "attention_mask": attention_mask,
        }
    )
    return model_inputs

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder(decoder)

Sets the decoder for the LlamaForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the LlamaForCausalLM class. TYPE: LlamaForCausalLM
decoder	The decoder object to be set for the model.

RETURNS	DESCRIPTION
	None.

This method sets the decoder object provided as an argument to the 'model' attribute of the LlamaForCausalLM instance. The 'model' attribute represents the decoder used for the causal language modeling task.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def set_decoder(self, decoder):
    """
    Sets the decoder for the LlamaForCausalLM model.

    Args:
        self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
        decoder: The decoder object to be set for the model.

    Returns:
        None.

    Raises:
        None.

    This method sets the decoder object provided as an argument to the 'model' attribute of the
    LlamaForCausalLM instance.
    The 'model' attribute represents the decoder used for the causal language modeling task.
    """
    self.model = decoder

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings(value)

Description: Sets the input embeddings of the LlamaForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the LlamaForCausalLM class. Type: LlamaForCausalLM Purpose: Represents the current instance of the LlamaForCausalLM class. Restrictions: Must be an instance of the LlamaForCausalLM class. TYPE: LlamaForCausalLM
value	The input embeddings to be set for the model. Type: Any Purpose: Represents the new input embeddings to be assigned to the model. Restrictions: None

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def set_input_embeddings(self, value):
    """
    Method: set_input_embeddings

    Description: Sets the input embeddings of the LlamaForCausalLM model.

    Args:
        self (LlamaForCausalLM):
            The instance of the LlamaForCausalLM class.

            - Type: LlamaForCausalLM
            - Purpose: Represents the current instance of the LlamaForCausalLM class.
            - Restrictions: Must be an instance of the LlamaForCausalLM class.

        value:
            The input embeddings to be set for the model.

            - Type: Any
            - Purpose: Represents the new input embeddings to be assigned to the model.
            - Restrictions: None

    Returns:
        None.

    Raises:
        None
    """
    self.model.embed_tokens = value

mindnlp.transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings(new_embeddings)

Sets the output embeddings for the LlamaForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the LlamaForCausalLM class. TYPE: LlamaForCausalLM
new_embeddings	The new embeddings to be set for the model's lm_head. TYPE: Tensor

RETURNS	DESCRIPTION
	None.

This method allows the user to update the output embeddings of the LlamaForCausalLM model by replacing the current embeddings with the provided new_embeddings. The new_embeddings should be a tensor of the same shape and size as the current embeddings. This method is useful in fine-tuning the model with custom embeddings or when transferring the model to a different task that requires different output embeddings.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def set_output_embeddings(self, new_embeddings):
    """
    Sets the output embeddings for the LlamaForCausalLM model.

    Args:
        self (LlamaForCausalLM): The instance of the LlamaForCausalLM class.
        new_embeddings (Tensor): The new embeddings to be set for the model's lm_head.

    Returns:
        None.

    Raises:
        None.

    This method allows the user to update the output embeddings of the LlamaForCausalLM model by replacing the
    current embeddings with the provided new_embeddings.
    The new_embeddings should be a tensor of the same shape and size as the current embeddings.
    This method is useful in fine-tuning the model with custom embeddings or when transferring the model to
    a different task that requires different output embeddings.
    """
    self.lm_head = new_embeddings

mindnlp.transformers.models.llama.modeling_llama.LlamaForSequenceClassification

Bases: LlamaPreTrainedModel

LlamaForSequenceClassification

This class is a sequence classification model based on the Llama architecture. It inherits from the LlamaPreTrainedModel class.

ATTRIBUTE	DESCRIPTION
num_labels	The number of labels for the sequence classification task. TYPE: int
model	The LlamaModel instance used for the sequence classification. TYPE: LlamaModel
score	The final layer that computes the logits for the classification. TYPE: Linear

METHOD	DESCRIPTION
__init__	Initializes a new instance of the LlamaForSequenceClassification class.
get_input_embeddings	Retrieves the input embeddings from the LlamaModel.
set_input_embeddings	Sets the input embeddings in the LlamaModel.
forward	forwards the sequence classification model. Parameters: input_ids (mindspore.Tensor): The input tensor of shape (batch_size, sequence_length). attention_mask (Optional[mindspore.Tensor]): The attention mask tensor of shape (batch_size, sequence_length). position_ids (Optional[mindspore.Tensor]): The position IDs tensor of shape (batch_size, sequence_length). past_key_values (Optional[List[mindspore.Tensor]]): The list of past key-value tensors. inputs_embeds (Optional[mindspore.Tensor]): The input embeddings tensor of shape (batch_size, sequence_length, hidden_size). labels (Optional[mindspore.Tensor]): The labels tensor of shape (batch_size,). use_cache (Optional[bool]): Whether to use cache for the model. output_attentions (Optional[bool]): Whether to output attention tensors. output_hidden_states (Optional[bool]): Whether to output hidden state tensors. return_dict (Optional[bool]): Whether to return a SequenceClassifierOutputWithPast object. Returns: Union[Tuple, SequenceClassifierOutputWithPast]: The output tuple or a SequenceClassifierOutputWithPast object.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaForSequenceClassification(LlamaPreTrainedModel):

    """
    LlamaForSequenceClassification

    This class is a sequence classification model based on the Llama architecture. It inherits from the LlamaPreTrainedModel class.

    Attributes:
        num_labels (int): The number of labels for the sequence classification task.
        model (LlamaModel): The LlamaModel instance used for the sequence classification.
        score (nn.Linear): The final layer that computes the logits for the classification.

    Methods:
        __init__:
            Initializes a new instance of the LlamaForSequenceClassification class.

        get_input_embeddings:
            Retrieves the input embeddings from the LlamaModel.

        set_input_embeddings:
            Sets the input embeddings in the LlamaModel.

        forward:
            forwards the sequence classification model.

            Parameters:

            - input_ids (mindspore.Tensor): The input tensor of shape `(batch_size, sequence_length)`.
            - attention_mask (Optional[mindspore.Tensor]): The attention mask tensor of shape
            `(batch_size, sequence_length)`.
            - position_ids (Optional[mindspore.Tensor]): The position IDs tensor of shape
            `(batch_size, sequence_length)`.
            - past_key_values (Optional[List[mindspore.Tensor]]): The list of past key-value tensors.
            - inputs_embeds (Optional[mindspore.Tensor]): The input embeddings tensor of shape
            `(batch_size, sequence_length, hidden_size)`.
            - labels (Optional[mindspore.Tensor]): The labels tensor of shape `(batch_size,)`.
            - use_cache (Optional[bool]): Whether to use cache for the model.
            - output_attentions (Optional[bool]): Whether to output attention tensors.
            - output_hidden_states (Optional[bool]): Whether to output hidden state tensors.
            - return_dict (Optional[bool]): Whether to return a SequenceClassifierOutputWithPast object.

            Returns:

            - Union[Tuple, SequenceClassifierOutputWithPast]:
            The output tuple or a SequenceClassifierOutputWithPast object.
    """
    def __init__(self, config):
        """
        Initializes an instance of the LlamaForSequenceClassification class.

        Args:
            self: The instance of the class.
            config: An object of type 'Config',
                containing the configuration parameters for the model.

                - Type: 'Config' object
                - Purpose: The configuration parameters for the model
                - Restrictions: None

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.num_labels = config.num_labels
        self.model = LlamaModel(config)
        self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Returns the input embeddings of the given sequence for the LlamaForSequenceClassification model.

        Args:
            self: An instance of the LlamaForSequenceClassification class.

        Returns:
            embed_tokens: The method returns a value of type 'None'.

        Raises:
            None.

        This method retrieves the input embeddings for the given sequence from the LlamaForSequenceClassification model.
        Input embeddings are the vector representations of the input tokens in the sequence that the model uses for
        further processing. These embeddings capture the contextual information of the tokens and are essential for
        downstream tasks such as sequence classification.

        Note:
            The input embeddings are obtained by calling the 'embed_tokens' method of the model instance.

        Example:
            ```python
            >>> llama_classifier = LlamaForSequenceClassification()
            >>> embeddings = llama_classifier.get_input_embeddings()
            ```
        """
        return self.model.embed_tokens

    def set_input_embeddings(self, value):
        """Set the embedding layer of the LlamaForSequenceClassification model with a specified value.

        Args:
            self (LlamaForSequenceClassification): An instance of the LlamaForSequenceClassification class.
            value (torch.nn.Embedding): The embedding layer to be set in the model.

        Returns:
            None.

        Raises:
            TypeError: If the value parameter is not an instance of torch.nn.Embedding.
        """
        self.model.embed_tokens = value

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
                `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        transformer_outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        hidden_states = transformer_outputs[0]
        logits = self.score(hidden_states)

        if input_ids is not None:
            batch_size, _ = input_ids.shape[:2]
        else:
            batch_size, _ = inputs_embeds.shape[:2]

        if self.config.pad_token_id is None and batch_size != 1:
            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
        if self.config.pad_token_id is None:
            sequence_lengths = -1
        else:
            if input_ids is not None:
                sequence_lengths = ops.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
                sequence_lengths = sequence_lengths % input_ids.shape[-1]
            else:
                sequence_lengths = -1

        pooled_logits = logits[ops.arange(batch_size), sequence_lengths]

        loss = None
        if labels is not None:
            if self.config.problem_type is None:
                if self.num_labels == 1:
                    self.config.problem_type = "regression"
                elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                    self.config.problem_type = "single_label_classification"
                else:
                    self.config.problem_type = "multi_label_classification"

            if self.config.problem_type == "regression":
                if self.num_labels == 1:
                    loss = F.mse_loss(pooled_logits.squeeze(), labels.squeeze())
                else:
                    loss = F.mse_loss(pooled_logits, labels)
            elif self.config.problem_type == "single_label_classification":
                loss = F.cross_entropy(pooled_logits.view(-1, self.num_labels), labels.view(-1))
            elif self.config.problem_type == "multi_label_classification":
                loss = F.binary_cross_entropy_with_logits(pooled_logits, labels)
        if not return_dict:
            output = (pooled_logits,) + transformer_outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutputWithPast(
            loss=loss,
            logits=pooled_logits,
            past_key_values=transformer_outputs.past_key_values,
            hidden_states=transformer_outputs.hidden_states,
            attentions=transformer_outputs.attentions,
        )

mindnlp.transformers.models.llama.modeling_llama.LlamaForSequenceClassification.__init__(config)

Initializes an instance of the LlamaForSequenceClassification class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of type 'Config', containing the configuration parameters for the model. Type: 'Config' object Purpose: The configuration parameters for the model Restrictions: None

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config):
    """
    Initializes an instance of the LlamaForSequenceClassification class.

    Args:
        self: The instance of the class.
        config: An object of type 'Config',
            containing the configuration parameters for the model.

            - Type: 'Config' object
            - Purpose: The configuration parameters for the model
            - Restrictions: None

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.num_labels = config.num_labels
    self.model = LlamaModel(config)
    self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.llama.modeling_llama.LlamaForSequenceClassification.forward(input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If config.num_labels > 1 a classification loss is computed (Cross-Entropy). TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, SequenceClassifierOutputWithPast]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    transformer_outputs = self.model(
        input_ids,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    hidden_states = transformer_outputs[0]
    logits = self.score(hidden_states)

    if input_ids is not None:
        batch_size, _ = input_ids.shape[:2]
    else:
        batch_size, _ = inputs_embeds.shape[:2]

    if self.config.pad_token_id is None and batch_size != 1:
        raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
    if self.config.pad_token_id is None:
        sequence_lengths = -1
    else:
        if input_ids is not None:
            sequence_lengths = ops.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
            sequence_lengths = sequence_lengths % input_ids.shape[-1]
        else:
            sequence_lengths = -1

    pooled_logits = logits[ops.arange(batch_size), sequence_lengths]

    loss = None
    if labels is not None:
        if self.config.problem_type is None:
            if self.num_labels == 1:
                self.config.problem_type = "regression"
            elif self.num_labels > 1 and labels.dtype in (mindspore.int64, mindspore.int32):
                self.config.problem_type = "single_label_classification"
            else:
                self.config.problem_type = "multi_label_classification"

        if self.config.problem_type == "regression":
            if self.num_labels == 1:
                loss = F.mse_loss(pooled_logits.squeeze(), labels.squeeze())
            else:
                loss = F.mse_loss(pooled_logits, labels)
        elif self.config.problem_type == "single_label_classification":
            loss = F.cross_entropy(pooled_logits.view(-1, self.num_labels), labels.view(-1))
        elif self.config.problem_type == "multi_label_classification":
            loss = F.binary_cross_entropy_with_logits(pooled_logits, labels)
    if not return_dict:
        output = (pooled_logits,) + transformer_outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutputWithPast(
        loss=loss,
        logits=pooled_logits,
        past_key_values=transformer_outputs.past_key_values,
        hidden_states=transformer_outputs.hidden_states,
        attentions=transformer_outputs.attentions,
    )

mindnlp.transformers.models.llama.modeling_llama.LlamaForSequenceClassification.get_input_embeddings()

Returns the input embeddings of the given sequence for the LlamaForSequenceClassification model.

PARAMETER	DESCRIPTION
self	An instance of the LlamaForSequenceClassification class.

RETURNS	DESCRIPTION
embed_tokens	The method returns a value of type 'None'.

This method retrieves the input embeddings for the given sequence from the LlamaForSequenceClassification model. Input embeddings are the vector representations of the input tokens in the sequence that the model uses for further processing. These embeddings capture the contextual information of the tokens and are essential for downstream tasks such as sequence classification.

Note

The input embeddings are obtained by calling the 'embed_tokens' method of the model instance.

Example

>>> llama_classifier = LlamaForSequenceClassification()
>>> embeddings = llama_classifier.get_input_embeddings()

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def get_input_embeddings(self):
    """
    Returns the input embeddings of the given sequence for the LlamaForSequenceClassification model.

    Args:
        self: An instance of the LlamaForSequenceClassification class.

    Returns:
        embed_tokens: The method returns a value of type 'None'.

    Raises:
        None.

    This method retrieves the input embeddings for the given sequence from the LlamaForSequenceClassification model.
    Input embeddings are the vector representations of the input tokens in the sequence that the model uses for
    further processing. These embeddings capture the contextual information of the tokens and are essential for
    downstream tasks such as sequence classification.

    Note:
        The input embeddings are obtained by calling the 'embed_tokens' method of the model instance.

    Example:
        ```python
        >>> llama_classifier = LlamaForSequenceClassification()
        >>> embeddings = llama_classifier.get_input_embeddings()
        ```
    """
    return self.model.embed_tokens

mindnlp.transformers.models.llama.modeling_llama.LlamaForSequenceClassification.set_input_embeddings(value)

Set the embedding layer of the LlamaForSequenceClassification model with a specified value.

PARAMETER	DESCRIPTION
self	An instance of the LlamaForSequenceClassification class. TYPE: LlamaForSequenceClassification
value	The embedding layer to be set in the model. TYPE: Embedding

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the value parameter is not an instance of torch.nn.Embedding.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def set_input_embeddings(self, value):
    """Set the embedding layer of the LlamaForSequenceClassification model with a specified value.

    Args:
        self (LlamaForSequenceClassification): An instance of the LlamaForSequenceClassification class.
        value (torch.nn.Embedding): The embedding layer to be set in the model.

    Returns:
        None.

    Raises:
        TypeError: If the value parameter is not an instance of torch.nn.Embedding.
    """
    self.model.embed_tokens = value

mindnlp.transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding

Bases: LlamaRotaryEmbedding

LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding):
    """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
    def __init__(self, dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0):
        """
        Initializes a new instance of the LlamaLinearScalingRotaryEmbedding class.

        Args:
            self (LlamaLinearScalingRotaryEmbedding): The current instance of the class.
            dim (int): The dimensionality of the embedding.
            max_position_embeddings (int, optional): The maximum number of position embeddings. Default is 2048.
            base (int, optional): The base value used for scaling. Default is 10000.
            scaling_factor (float, optional): The scaling factor applied to the embeddings. Default is 1.0.

        Returns:
            None.

        Raises:
            None.
        """
        self.scaling_factor = scaling_factor
        super().__init__(dim, max_position_embeddings, base)

    def _set_cos_sin_cache(self, seq_len, dtype):
        """
        Sets the cosine and sine caches for the LlamaLinearScalingRotaryEmbedding class.

        Args:
            self (LlamaLinearScalingRotaryEmbedding): The instance of the LlamaLinearScalingRotaryEmbedding class.
            seq_len (int): The length of the sequence.
            dtype: The desired data type for the cache.

        Returns:
            None.

        Raises:
            None.
        """
        self.max_seq_len_cached = seq_len
        t = ops.arange(self.max_seq_len_cached, dtype=self.inv_freq.dtype)
        t = ops.div(t, self.scaling_factor)

        freqs = ops.outer(t, self.inv_freq)
        # Different from paper, but it uses a different permutation in order to obtain the same calculation
        emb = ops.cat((freqs, freqs), dim=-1)
        self.cos_cached = ops.cos(emb).to(dtype)
        self.sin_cached = ops.sin(emb).to(dtype)

mindnlp.transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding.__init__(dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0)

Initializes a new instance of the LlamaLinearScalingRotaryEmbedding class.

PARAMETER	DESCRIPTION
self	The current instance of the class. TYPE: LlamaLinearScalingRotaryEmbedding
dim	The dimensionality of the embedding. TYPE: int
max_position_embeddings	The maximum number of position embeddings. Default is 2048. TYPE: int DEFAULT: 2048
base	The base value used for scaling. Default is 10000. TYPE: int DEFAULT: 10000
scaling_factor	The scaling factor applied to the embeddings. Default is 1.0. TYPE: float DEFAULT: 1.0

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, dim, max_position_embeddings=2048, base=10000, scaling_factor=1.0):
    """
    Initializes a new instance of the LlamaLinearScalingRotaryEmbedding class.

    Args:
        self (LlamaLinearScalingRotaryEmbedding): The current instance of the class.
        dim (int): The dimensionality of the embedding.
        max_position_embeddings (int, optional): The maximum number of position embeddings. Default is 2048.
        base (int, optional): The base value used for scaling. Default is 10000.
        scaling_factor (float, optional): The scaling factor applied to the embeddings. Default is 1.0.

    Returns:
        None.

    Raises:
        None.
    """
    self.scaling_factor = scaling_factor
    super().__init__(dim, max_position_embeddings, base)

mindnlp.transformers.models.llama.modeling_llama.LlamaMLP

Bases: Module

This class represents a multi-layer perceptron (MLP) model called LlamaMLP.

LlamaMLP inherits from the nn.Module class and is designed for deep learning tasks. It consists of multiple layers, including gate projection, up projection, and down projection layers, which are used to transform the input data and produce the final output.

ATTRIBUTE	DESCRIPTION
config	The configuration object that stores the hyperparameters of the LlamaMLP model. TYPE: object
hidden_size	The size of the hidden layer in the LlamaMLP model. TYPE: int
intermediate_size	The size of the intermediate layer in the LlamaMLP model. TYPE: int
gate_proj	The dense layer responsible for the gate projection in the LlamaMLP model. TYPE: object
up_proj	The dense layer responsible for the up projection in the LlamaMLP model. TYPE: object
down_proj	The dense layer responsible for the down projection in the LlamaMLP model. TYPE: object
act_fn	The activation function used in the LlamaMLP model. TYPE: function

METHOD	DESCRIPTION
__init__	Initializes a new instance of the LlamaMLP class.
forward	forwards the LlamaMLP model by applying the necessary transformations on the input data. This method returns the final output of the LlamaMLP model.

Note

The LlamaMLP model supports pretraining when the 'pretraining_tp' hyperparameter is greater than 1. In this case, the input data is split into slices to perform parallel computations. Otherwise, the computations are performed in a single path.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaMLP(nn.Module):

    """
    This class represents a multi-layer perceptron (MLP) model called LlamaMLP.

    LlamaMLP inherits from the nn.Module class and is designed for deep learning tasks.
    It consists of multiple layers, including gate projection, up projection, and down projection layers,
    which are used to transform the input data and produce the final output.

    Attributes:
        config (object): The configuration object that stores the hyperparameters of the LlamaMLP model.
        hidden_size (int): The size of the hidden layer in the LlamaMLP model.
        intermediate_size (int): The size of the intermediate layer in the LlamaMLP model.
        gate_proj (object): The dense layer responsible for the gate projection in the LlamaMLP model.
        up_proj (object): The dense layer responsible for the up projection in the LlamaMLP model.
        down_proj (object): The dense layer responsible for the down projection in the LlamaMLP model.
        act_fn (function): The activation function used in the LlamaMLP model.

    Methods:
        __init__:
            Initializes a new instance of the LlamaMLP class.

        forward:
            forwards the LlamaMLP model by applying the necessary transformations on the input data.
            This method returns the final output of the LlamaMLP model.

    Note:
        The LlamaMLP model supports pretraining when the 'pretraining_tp' hyperparameter is greater than 1.
        In this case, the input data is split into slices to perform parallel computations. Otherwise, the
        computations are performed in a single path.
    """
    def __init__(self, config):
        """
        Initializes an instance of the LlamaMLP class.

        Args:
            self: The instance of the class.
            config: An object of type 'Config' containing the configuration settings for the MLP.
                The 'Config' object should have the following properties:

                - hidden_size (int): The size of the hidden layer.
                - intermediate_size (int): The size of the intermediate layer.
                - hidden_act (str): The activation function for the hidden layer.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()
        self.config = config
        self.hidden_size = config.hidden_size
        self.intermediate_size = config.intermediate_size
        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
        self.act_fn = ACT2FN[config.hidden_act]

    def forward(self, x):
        """
        forwards the output of the LlamaMLP model based on the input and configuration settings.

        Args:
            self (LlamaMLP): The instance of the LlamaMLP class.
            x (tensor): The input tensor to be processed by the model.

        Returns:
            None.

        Raises:
            ValueError: If the value of 'pretraining_tp' in the configuration is less than or equal to 1.
            TypeError: If any of the operations cannot be performed due to data type mismatch or other reasons.
            IndexError: If any index used for slicing or accessing tensors is out of bounds.
        """
        if self.config.pretraining_tp > 1:
            slices = self.intermediate_size // self.config.pretraining_tp
            gate_proj_slices = self.gate_proj.weight.split(slices, axis=0)
            up_proj_slices = self.up_proj.weight.split(slices, axis=0)
            down_proj_slices = self.down_proj.weight.split(slices, axis=1)

            gate_proj = ops.cat(
                [F.linear(x, gate_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1
            )
            up_proj = ops.cat([F.linear(x, up_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1)

            intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, axis=2)
            down_proj = [
                F.linear(intermediate_states[i], down_proj_slices[i]) for i in range(self.config.pretraining_tp)
            ]
            down_proj = sum(down_proj)
        else:
            down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        return down_proj

mindnlp.transformers.models.llama.modeling_llama.LlamaMLP.__init__(config)

Initializes an instance of the LlamaMLP class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An object of type 'Config' containing the configuration settings for the MLP. The 'Config' object should have the following properties: hidden_size (int): The size of the hidden layer. intermediate_size (int): The size of the intermediate layer. hidden_act (str): The activation function for the hidden layer.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config):
    """
    Initializes an instance of the LlamaMLP class.

    Args:
        self: The instance of the class.
        config: An object of type 'Config' containing the configuration settings for the MLP.
            The 'Config' object should have the following properties:

            - hidden_size (int): The size of the hidden layer.
            - intermediate_size (int): The size of the intermediate layer.
            - hidden_act (str): The activation function for the hidden layer.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()
    self.config = config
    self.hidden_size = config.hidden_size
    self.intermediate_size = config.intermediate_size
    self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
    self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
    self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
    self.act_fn = ACT2FN[config.hidden_act]

mindnlp.transformers.models.llama.modeling_llama.LlamaMLP.forward(x)

forwards the output of the LlamaMLP model based on the input and configuration settings.

PARAMETER	DESCRIPTION
self	The instance of the LlamaMLP class. TYPE: LlamaMLP
x	The input tensor to be processed by the model. TYPE: tensor

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the value of 'pretraining_tp' in the configuration is less than or equal to 1.
TypeError	If any of the operations cannot be performed due to data type mismatch or other reasons.
IndexError	If any index used for slicing or accessing tensors is out of bounds.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(self, x):
    """
    forwards the output of the LlamaMLP model based on the input and configuration settings.

    Args:
        self (LlamaMLP): The instance of the LlamaMLP class.
        x (tensor): The input tensor to be processed by the model.

    Returns:
        None.

    Raises:
        ValueError: If the value of 'pretraining_tp' in the configuration is less than or equal to 1.
        TypeError: If any of the operations cannot be performed due to data type mismatch or other reasons.
        IndexError: If any index used for slicing or accessing tensors is out of bounds.
    """
    if self.config.pretraining_tp > 1:
        slices = self.intermediate_size // self.config.pretraining_tp
        gate_proj_slices = self.gate_proj.weight.split(slices, axis=0)
        up_proj_slices = self.up_proj.weight.split(slices, axis=0)
        down_proj_slices = self.down_proj.weight.split(slices, axis=1)

        gate_proj = ops.cat(
            [F.linear(x, gate_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1
        )
        up_proj = ops.cat([F.linear(x, up_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1)

        intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, axis=2)
        down_proj = [
            F.linear(intermediate_states[i], down_proj_slices[i]) for i in range(self.config.pretraining_tp)
        ]
        down_proj = sum(down_proj)
    else:
        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

    return down_proj

mindnlp.transformers.models.llama.modeling_llama.LlamaModel

Bases: LlamaPreTrainedModel

Transformer decoder consisting of config.num_hidden_layers layers. Each layer is a [LlamaDecoderLayer]

PARAMETER	DESCRIPTION
config	LlamaConfig TYPE: LlamaConfig

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaModel(LlamaPreTrainedModel):
    """
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]

    Args:
        config: LlamaConfig
    """
    def __init__(self, config: LlamaConfig):
        """
        Initializes a new instance of the LlamaModel class.

        Args:
            self: The object instance.
            config (LlamaConfig): The configuration object for the LlamaModel.
                This parameter specifies the configuration settings for the model.
                It should be an instance of the LlamaConfig class.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.padding_idx = config.pad_token_id
        self.vocab_size = config.vocab_size

        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx)
        self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Method: get_input_embeddings

        Description:
            This method retrieves the input embeddings from the LlamaModel instance.

        Args:
            self (LlamaModel): The LlamaModel instance that this method is called on.

        Returns:
            None: This method returns the embed_tokens attribute of the LlamaModel instance,
                which represents the input embeddings. The return value is of type None.

        Raises:
            None.
        """
        return self.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the LlamaModel instance.

        Args:
            self (LlamaModel): The LlamaModel instance.
            value (torch.Tensor): The input embeddings to be set.
                It should be a tensor of shape (num_embeddings, embedding_dim).

        Returns:
            None

        Raises:
            TypeError: If the input value is not a tensor.
            ValueError: If the input tensor shape is invalid.
        """
        self.embed_tokens = value

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        position_ids: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[List[mindspore.Tensor]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, BaseModelOutputWithPast]:
        """
        forwards the LlamaModel.

        Args:
            self (LlamaModel): The instance of the LlamaModel class.
            input_ids (mindspore.Tensor, optional): The input IDs tensor. Default is None.
            attention_mask (mindspore.Tensor, optional): The attention mask tensor. Default is None.
            position_ids (mindspore.Tensor, optional): The position IDs tensor. Default is None.
            past_key_values (List[mindspore.Tensor], optional): The list of past key values. Default is None.
            inputs_embeds (mindspore.Tensor, optional): The input embeddings tensor. Default is None.
            use_cache (bool, optional): Whether to use cache. Default is None.
            output_attentions (bool, optional): Whether to output attentions. Default is None.
            output_hidden_states (bool, optional): Whether to output hidden states. Default is None.
            return_dict (bool, optional): Whether to return a dictionary. Default is None.

        Returns:
            Union[Tuple, BaseModelOutputWithPast]: The output of the LlamaModel.
                It can be a tuple containing hidden states, next cache, all hidden states, and all self attentions;
                or an instance of BaseModelOutputWithPast.

        Raises:
            ValueError: If both input_ids and inputs_embeds are specified.
            ValueError: If neither input_ids nor inputs_embeds are specified.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache

        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        if input_ids is not None:
            batch_size, seq_length = input_ids.shape[:2]
        elif inputs_embeds is not None:
            batch_size, seq_length = inputs_embeds.shape[:2]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        past_key_values_length = 0
        if past_key_values is not None:
            past_key_values_length = past_key_values[0][0].shape[2]

        if position_ids is None:
            position_ids = ops.arange(
                past_key_values_length, seq_length + past_key_values_length, dtype=mindspore.int64
            )
            position_ids = position_ids.unsqueeze(0)

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        # 4d mask is passed through the layers
        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
        )

        # embed positions
        hidden_states = inputs_embeds

        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        next_decoder_cache = () if use_cache else None

        for idx, decoder_layer in enumerate(self.layers):
            if output_hidden_states:
                all_hidden_states += (hidden_states,)

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                position_ids=position_ids,
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )

            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

        hidden_states = self.norm(hidden_states)

        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
        return BaseModelOutputWithPast(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
        )

mindnlp.transformers.models.llama.modeling_llama.LlamaModel.__init__(config)

Initializes a new instance of the LlamaModel class.

PARAMETER	DESCRIPTION
self	The object instance.
config	The configuration object for the LlamaModel. This parameter specifies the configuration settings for the model. It should be an instance of the LlamaConfig class. TYPE: LlamaConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, config: LlamaConfig):
    """
    Initializes a new instance of the LlamaModel class.

    Args:
        self: The object instance.
        config (LlamaConfig): The configuration object for the LlamaModel.
            This parameter specifies the configuration settings for the model.
            It should be an instance of the LlamaConfig class.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.padding_idx = config.pad_token_id
    self.vocab_size = config.vocab_size

    self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx)
    self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
    self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.llama.modeling_llama.LlamaModel.forward(input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

forwards the LlamaModel.

PARAMETER	DESCRIPTION
self	The instance of the LlamaModel class. TYPE: LlamaModel
input_ids	The input IDs tensor. Default is None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask tensor. Default is None. TYPE: Tensor DEFAULT: None
position_ids	The position IDs tensor. Default is None. TYPE: Tensor DEFAULT: None
past_key_values	The list of past key values. Default is None. TYPE: List[Tensor] DEFAULT: None
inputs_embeds	The input embeddings tensor. Default is None. TYPE: Tensor DEFAULT: None
use_cache	Whether to use cache. Default is None. TYPE: bool DEFAULT: None
output_attentions	Whether to output attentions. Default is None. TYPE: bool DEFAULT: None
output_hidden_states	Whether to output hidden states. Default is None. TYPE: bool DEFAULT: None
return_dict	Whether to return a dictionary. Default is None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, BaseModelOutputWithPast]	Union[Tuple, BaseModelOutputWithPast]: The output of the LlamaModel. It can be a tuple containing hidden states, next cache, all hidden states, and all self attentions; or an instance of BaseModelOutputWithPast.

RAISES	DESCRIPTION
ValueError	If both input_ids and inputs_embeds are specified.
ValueError	If neither input_ids nor inputs_embeds are specified.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    position_ids: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[List[mindspore.Tensor]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, BaseModelOutputWithPast]:
    """
    forwards the LlamaModel.

    Args:
        self (LlamaModel): The instance of the LlamaModel class.
        input_ids (mindspore.Tensor, optional): The input IDs tensor. Default is None.
        attention_mask (mindspore.Tensor, optional): The attention mask tensor. Default is None.
        position_ids (mindspore.Tensor, optional): The position IDs tensor. Default is None.
        past_key_values (List[mindspore.Tensor], optional): The list of past key values. Default is None.
        inputs_embeds (mindspore.Tensor, optional): The input embeddings tensor. Default is None.
        use_cache (bool, optional): Whether to use cache. Default is None.
        output_attentions (bool, optional): Whether to output attentions. Default is None.
        output_hidden_states (bool, optional): Whether to output hidden states. Default is None.
        return_dict (bool, optional): Whether to return a dictionary. Default is None.

    Returns:
        Union[Tuple, BaseModelOutputWithPast]: The output of the LlamaModel.
            It can be a tuple containing hidden states, next cache, all hidden states, and all self attentions;
            or an instance of BaseModelOutputWithPast.

    Raises:
        ValueError: If both input_ids and inputs_embeds are specified.
        ValueError: If neither input_ids nor inputs_embeds are specified.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache

    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    if input_ids is not None:
        batch_size, seq_length = input_ids.shape[:2]
    elif inputs_embeds is not None:
        batch_size, seq_length = inputs_embeds.shape[:2]
    else:
        raise ValueError("You have to specify either input_ids or inputs_embeds")

    past_key_values_length = 0
    if past_key_values is not None:
        past_key_values_length = past_key_values[0][0].shape[2]

    if position_ids is None:
        position_ids = ops.arange(
            past_key_values_length, seq_length + past_key_values_length, dtype=mindspore.int64
        )
        position_ids = position_ids.unsqueeze(0)

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    # 4d mask is passed through the layers
    attention_mask = _prepare_4d_causal_attention_mask(
        attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
    )

    # embed positions
    hidden_states = inputs_embeds

    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None
    next_decoder_cache = () if use_cache else None

    for idx, decoder_layer in enumerate(self.layers):
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        past_key_value = past_key_values[idx] if past_key_values is not None else None

        layer_outputs = decoder_layer(
            hidden_states,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_value=past_key_value,
            output_attentions=output_attentions,
            use_cache=use_cache,
        )

        hidden_states = layer_outputs[0]

        if use_cache:
            next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)

        if output_attentions:
            all_self_attns += (layer_outputs[1],)

    hidden_states = self.norm(hidden_states)

    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    next_cache = next_decoder_cache if use_cache else None
    if not return_dict:
        return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
    return BaseModelOutputWithPast(
        last_hidden_state=hidden_states,
        past_key_values=next_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
    )

mindnlp.transformers.models.llama.modeling_llama.LlamaModel.get_input_embeddings()

Description

This method retrieves the input embeddings from the LlamaModel instance.

PARAMETER	DESCRIPTION
self	The LlamaModel instance that this method is called on. TYPE: LlamaModel

RETURNS	DESCRIPTION
None	This method returns the embed_tokens attribute of the LlamaModel instance, which represents the input embeddings. The return value is of type None.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def get_input_embeddings(self):
    """
    Method: get_input_embeddings

    Description:
        This method retrieves the input embeddings from the LlamaModel instance.

    Args:
        self (LlamaModel): The LlamaModel instance that this method is called on.

    Returns:
        None: This method returns the embed_tokens attribute of the LlamaModel instance,
            which represents the input embeddings. The return value is of type None.

    Raises:
        None.
    """
    return self.embed_tokens

mindnlp.transformers.models.llama.modeling_llama.LlamaModel.set_input_embeddings(value)

Sets the input embeddings for the LlamaModel instance.

PARAMETER	DESCRIPTION
self	The LlamaModel instance. TYPE: LlamaModel
value	The input embeddings to be set. It should be a tensor of shape (num_embeddings, embedding_dim). TYPE: Tensor

RETURNS	DESCRIPTION
	None

RAISES	DESCRIPTION
TypeError	If the input value is not a tensor.
ValueError	If the input tensor shape is invalid.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the LlamaModel instance.

    Args:
        self (LlamaModel): The LlamaModel instance.
        value (torch.Tensor): The input embeddings to be set.
            It should be a tensor of shape (num_embeddings, embedding_dim).

    Returns:
        None

    Raises:
        TypeError: If the input value is not a tensor.
        ValueError: If the input tensor shape is invalid.
    """
    self.embed_tokens = value

mindnlp.transformers.models.llama.modeling_llama.LlamaPreTrainedModel

Bases: PreTrainedModel

LlamaPreTrainedModel is a Python class representing a pre-trained model for llama-based machine learning tasks. This class inherits from PreTrainedModel and provides methods for initializing weights.

The _init_weights method initializes the weights for the given cell. If the cell is of type nn.Linear, the weight is initialized using the Normal initializer within the specified range. If the cell has bias, it is initialized with zeros. If the cell is of type nn.Embedding, the weight is initialized with random normal values within the specified range, and the padding index is set to 0 if provided.

PARAMETER	DESCRIPTION
cell	The cell for which the weights need to be initialized.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaPreTrainedModel(PreTrainedModel):

    """
    LlamaPreTrainedModel is a Python class representing a pre-trained model for llama-based machine learning tasks.
    This class inherits from PreTrainedModel and provides methods for initializing weights.

    The _init_weights method initializes the weights for the given cell.
    If the cell is of type nn.Linear, the weight is initialized using the Normal initializer within the specified range.
    If the cell has bias, it is initialized with zeros.
    If the cell is of type nn.Embedding, the weight is initialized with random normal values within the specified range,
    and the padding index is set to 0 if provided.

    Parameters:
        cell: The cell for which the weights need to be initialized.

    Returns:
        None
    """
    config_class = LlamaConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = ["LlamaDecoderLayer"]
    _skip_keys_device_placement = "past_key_values"

    def _init_weights(self, cell):
        """Initialize the weights"""
        if isinstance(cell, nn.Linear):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            cell.weight.set_data(initializer(Normal(self.config.initializer_range),
                                                    cell.weight.shape, cell.weight.dtype))
            if cell.bias is not None:
                cell.bias.set_data(initializer('zeros', cell.bias.shape, cell.bias.dtype))
        elif isinstance(cell, nn.Embedding):
            weight = np.random.normal(0.0, self.config.initializer_range, cell.weight.shape)
            if cell.padding_idx:
                weight[cell.padding_idx] = 0

            cell.weight.set_data(Tensor(weight, cell.weight.dtype))

mindnlp.transformers.models.llama.modeling_llama.LlamaRMSNorm

Bases: Module

LlamaRMSNorm is a class that represents a normalization layer, equivalent to T5LayerNorm, used in deep learning models. It inherits from the nn.Module class.

This class provides methods to initialize and apply RMS normalization to the input hidden states. The RMS normalization is calculated based on the variance of the hidden states and a weight parameter. The normalized hidden states are then multiplied by the weight parameter to obtain the final output.

ATTRIBUTE	DESCRIPTION
weight	The weight parameter used in the RMS normalization. TYPE: Parameter
variance_epsilon	The epsilon value added to the variance to avoid division by zero. TYPE: float

METHOD	DESCRIPTION
__init__	Initializes a new instance of the LlamaRMSNorm class.
forward	Applies RMS normalization to the input hidden states.

Example

>>> # Create an instance of LlamaRMSNorm
>>> norm = LlamaRMSNorm(hidden_size=256)
...
>>> # Apply RMS normalization to hidden states
>>> output = norm.forward(hidden_states)

Please note that the LlamaRMSNorm class is designed to be used as part of a neural network model and requires the MindSpore library for execution.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaRMSNorm(nn.Module):

    """
    LlamaRMSNorm is a class that represents a normalization layer, equivalent to T5LayerNorm,
    used in deep learning models. It inherits from the nn.Module class.

    This class provides methods to initialize and apply RMS normalization to the input hidden states.
    The RMS normalization is calculated based on the variance of the hidden states and a weight parameter.
    The normalized hidden states are then multiplied by the weight parameter to obtain the final output.

    Attributes:
        weight (mindspore.Parameter): The weight parameter used in the RMS normalization.
        variance_epsilon (float): The epsilon value added to the variance to avoid division by zero.

    Methods:
        __init__: Initializes a new instance of the LlamaRMSNorm class.
        forward: Applies RMS normalization to the input hidden states.

    Example:
        ```python
        >>> # Create an instance of LlamaRMSNorm
        >>> norm = LlamaRMSNorm(hidden_size=256)
        ...
        >>> # Apply RMS normalization to hidden states
        >>> output = norm.forward(hidden_states)
        ```
    Please note that the LlamaRMSNorm class is designed to be used as part of a neural network model and requires the
    MindSpore library for execution."""
    def __init__(self, hidden_size, eps=1e-6):
        """
        LlamaRMSNorm is equivalent to T5LayerNorm
        """
        super().__init__()
        self.weight = Parameter(ops.ones(hidden_size), 'weight')
        self.variance_epsilon = eps

    def forward(self, hidden_states):
        """forwards the RMS normalization of the hidden states.

        Args:
            self (LlamaRMSNorm): The instance of the LlamaRMSNorm class.
            hidden_states (Union[Tensor, ndarray]): The input hidden states to be normalized.
                Should be a tensor or numpy array of any shape.

        Returns:
            None: This method does not return any value. The normalization is applied in place.

        Raises:
            ValueError: If the input hidden_states is not a valid tensor or numpy array.
            RuntimeError: If an error occurs during the normalization process.
        """
        if not self.training and USE_PYBOOST:
            return F.rms_norm(hidden_states, self.weight, self.variance_epsilon)
        input_dtype = hidden_states.dtype
        hidden_states = hidden_states.to(mindspore.float32)
        variance = ops.mean(ops.pow(hidden_states, 2), -1, True)
        hidden_states = ops.mul(hidden_states, ops.rsqrt(variance + self.variance_epsilon))
        return ops.mul(self.weight.astype(input_dtype), hidden_states.astype(input_dtype))

mindnlp.transformers.models.llama.modeling_llama.LlamaRMSNorm.__init__(hidden_size, eps=1e-06)

LlamaRMSNorm is equivalent to T5LayerNorm

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, hidden_size, eps=1e-6):
    """
    LlamaRMSNorm is equivalent to T5LayerNorm
    """
    super().__init__()
    self.weight = Parameter(ops.ones(hidden_size), 'weight')
    self.variance_epsilon = eps

mindnlp.transformers.models.llama.modeling_llama.LlamaRMSNorm.forward(hidden_states)

forwards the RMS normalization of the hidden states.

PARAMETER	DESCRIPTION
self	The instance of the LlamaRMSNorm class. TYPE: LlamaRMSNorm
hidden_states	The input hidden states to be normalized. Should be a tensor or numpy array of any shape. TYPE: Union[Tensor, ndarray]

RETURNS	DESCRIPTION
None	This method does not return any value. The normalization is applied in place.

RAISES	DESCRIPTION
ValueError	If the input hidden_states is not a valid tensor or numpy array.
RuntimeError	If an error occurs during the normalization process.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(self, hidden_states):
    """forwards the RMS normalization of the hidden states.

    Args:
        self (LlamaRMSNorm): The instance of the LlamaRMSNorm class.
        hidden_states (Union[Tensor, ndarray]): The input hidden states to be normalized.
            Should be a tensor or numpy array of any shape.

    Returns:
        None: This method does not return any value. The normalization is applied in place.

    Raises:
        ValueError: If the input hidden_states is not a valid tensor or numpy array.
        RuntimeError: If an error occurs during the normalization process.
    """
    if not self.training and USE_PYBOOST:
        return F.rms_norm(hidden_states, self.weight, self.variance_epsilon)
    input_dtype = hidden_states.dtype
    hidden_states = hidden_states.to(mindspore.float32)
    variance = ops.mean(ops.pow(hidden_states, 2), -1, True)
    hidden_states = ops.mul(hidden_states, ops.rsqrt(variance + self.variance_epsilon))
    return ops.mul(self.weight.astype(input_dtype), hidden_states.astype(input_dtype))

mindnlp.transformers.models.llama.modeling_llama.LlamaRotaryEmbedding

Bases: Module

The LlamaRotaryEmbedding class represents a rotary positional embedding layer that can be used in neural network models. It inherits from the nn.Module class.

ATTRIBUTE	DESCRIPTION
dim	The dimension of the embedding. TYPE: int
max_position_embeddings	The maximum number of position embeddings. TYPE: int
base	The base value used for calculating inverse frequencies. TYPE: int
inv_freq	The tensor containing the inverse frequencies calculated based on the dim and base values. TYPE: Tensor
max_seq_len_cached	The maximum sequence length for which cosine and sine values are cached. TYPE: int
cos_cached	The cached cosine values for the positional embeddings. TYPE: Tensor
sin_cached	The cached sine values for the positional embeddings. TYPE: Tensor

METHOD	DESCRIPTION
__init__	Initializes a new instance of the LlamaRotaryEmbedding class.
_set_cos_sin_cache	Sets up the cosine and sine cache for a given sequence length and data type.
forward	forwards the positional embedding for the input tensor x.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

class LlamaRotaryEmbedding(nn.Module):

    """
    The `LlamaRotaryEmbedding` class represents a rotary positional embedding layer that can be used in
    neural network models. It inherits from the `nn.Module` class.

    Attributes:
        dim (int): The dimension of the embedding.
        max_position_embeddings (int): The maximum number of position embeddings.
        base (int): The base value used for calculating inverse frequencies.
        inv_freq (Tensor): The tensor containing the inverse frequencies calculated based on the `dim` and `base` values.
        max_seq_len_cached (int): The maximum sequence length for which cosine and sine values are cached.
        cos_cached (Tensor): The cached cosine values for the positional embeddings.
        sin_cached (Tensor): The cached sine values for the positional embeddings.

    Methods:
        __init__:
            Initializes a new instance of the `LlamaRotaryEmbedding` class.

        _set_cos_sin_cache:
            Sets up the cosine and sine cache for a given sequence length and data type.

        forward:
            forwards the positional embedding for the input tensor `x`.
    """
    def __init__(self, dim, max_position_embeddings=2048, base=10000):
        """
        Initializes a new instance of the LlamaRotaryEmbedding class.

        Args:
            self: The LlamaRotaryEmbedding object itself.
            dim (int): The dimension of the embedding.
            max_position_embeddings (int, optional): The maximum number of position embeddings. Defaults to 2048.
            base (int, optional): The base value for calculating the inverse frequency. Defaults to 10000.

        Returns:
            None

        Raises:
            None
        """
        super().__init__()

        self.dim = dim
        self.max_position_embeddings = max_position_embeddings
        self.base = base
        inv_freq = 1.0 / (self.base ** (ops.arange(0, self.dim, 2).float() / self.dim))
        self.register_buffer('inv_freq', inv_freq)

        self._set_cos_sin_cache(
            seq_len=max_position_embeddings, dtype=mindspore.float32
        )

    def _set_cos_sin_cache(self, seq_len, dtype):
        """
        Sets the cosine and sine caches for LlamaRotaryEmbedding.

        Args:
            self (LlamaRotaryEmbedding): An instance of the LlamaRotaryEmbedding class.
            seq_len (int): The length of the sequence.
            dtype: The data type of the cache.

        Returns:
            None: The method updates the 'cos_cached' and 'sin_cached' attributes of the LlamaRotaryEmbedding instance.

        Raises:
            None.

        Description:
            This method sets the cosine and sine caches for LlamaRotaryEmbedding.
            The caches are used in the forward pass of the neural network for efficient calculation of rotary
            position embeddings.

            The method first sets the maximum sequence length cached to the given sequence length.
            It then creates a tensor 't' using the 'arange' operation from the 'ops' module, with the same data type as
            'inv_freq'.

            Next, it calculates the element-wise product of 't' and 'inv_freq' using 'einsum' operation
            from the 'ops' module. The result is a tensor 'freqs' which represents the frequencies for each position in
            the sequence.

            To create the cache tensor, 'freqs' is concatenated with itself along the last axis using the 'cat'
            operation from the 'ops' module. The resulting tensor 'emb' has shape (seq_len, 2 * frequency_dim),
            where frequency_dim is the dimension of the 'inv_freq' tensor.

            Finally, the 'cos_cached' and 'sin_cached' attributes are updated by calculating the cosine and sine of
            each element in 'emb', respectively. The resulting tensors are converted to the given data type
            'dtype' using the 'to' method.

        Note:
            It is assumed that the 'inv_freq' attribute of the LlamaRotaryEmbedding instance has been initialized
            prior to calling this method.
        """
        self.max_seq_len_cached = seq_len
        t = ops.arange(self.max_seq_len_cached, dtype=self.inv_freq.dtype)

        freqs = ops.outer(t, self.inv_freq)
        # Different from paper, but it uses a different permutation in order to obtain the same calculation
        emb = ops.cat((freqs, freqs), dim=-1)
        self.cos_cached = ops.cos(emb).to(dtype)
        self.sin_cached = ops.sin(emb).to(dtype)

    def forward(self, x, seq_len=None):
        """
        forwards a subset of the cached cosine and sine values based on the given sequence length.

        Args:
            self (LlamaRotaryEmbedding): An instance of the LlamaRotaryEmbedding class.
            x: The input tensor.
            seq_len (int, optional): The length of the desired subset. Defaults to None.

        Returns:
            tuple: A tuple containing two tensors. The first tensor represents the subset of cached cosine values,
                and the second tensor represents the subset of cached sine values. Both tensors are of the
                same dtype as x.

        Raises:
            TypeError: If seq_len is not an integer or None.
            ValueError: If seq_len is less than or equal to 0.
            AttributeError: If seq_len exceeds the maximum sequence length that has been cached.

        Note:
            The returned subset will include elements up to the index 'seq_len - 1' from the cached cosine and sine values.
            If seq_len is None or not provided, the entire cached cosine and sine values will be returned.
        """
        # x: [bs, num_attention_heads, seq_len, head_size]
        if seq_len > self.max_seq_len_cached:
            self._set_cos_sin_cache(seq_len=seq_len, dtype=x.dtype)

        return (
            # 1. default tensor slice
            # self.cos_cached[:seq_len].to(dtype=x.dtype),
            # self.sin_cached[:seq_len].to(dtype=x.dtype),
            # 2. use stridedslice
            # ops.getitem(self.cos_cached, slice(None, seq_len, None)).to(dtype=x.dtype),
            # ops.getitem(self.sin_cached, slice(None, seq_len, None)).to(dtype=x.dtype),
            # 3. use pyboost narrow
            ops.narrow(self.cos_cached, 0, 0, seq_len).to(dtype=x.dtype),
            ops.narrow(self.sin_cached, 0, 0, seq_len).to(dtype=x.dtype),
        )

mindnlp.transformers.models.llama.modeling_llama.LlamaRotaryEmbedding.__init__(dim, max_position_embeddings=2048, base=10000)

Initializes a new instance of the LlamaRotaryEmbedding class.

PARAMETER	DESCRIPTION
self	The LlamaRotaryEmbedding object itself.
dim	The dimension of the embedding. TYPE: int
max_position_embeddings	The maximum number of position embeddings. Defaults to 2048. TYPE: int DEFAULT: 2048
base	The base value for calculating the inverse frequency. Defaults to 10000. TYPE: int DEFAULT: 10000

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def __init__(self, dim, max_position_embeddings=2048, base=10000):
    """
    Initializes a new instance of the LlamaRotaryEmbedding class.

    Args:
        self: The LlamaRotaryEmbedding object itself.
        dim (int): The dimension of the embedding.
        max_position_embeddings (int, optional): The maximum number of position embeddings. Defaults to 2048.
        base (int, optional): The base value for calculating the inverse frequency. Defaults to 10000.

    Returns:
        None

    Raises:
        None
    """
    super().__init__()

    self.dim = dim
    self.max_position_embeddings = max_position_embeddings
    self.base = base
    inv_freq = 1.0 / (self.base ** (ops.arange(0, self.dim, 2).float() / self.dim))
    self.register_buffer('inv_freq', inv_freq)

    self._set_cos_sin_cache(
        seq_len=max_position_embeddings, dtype=mindspore.float32
    )

mindnlp.transformers.models.llama.modeling_llama.LlamaRotaryEmbedding.forward(x, seq_len=None)

forwards a subset of the cached cosine and sine values based on the given sequence length.

PARAMETER	DESCRIPTION
self	An instance of the LlamaRotaryEmbedding class. TYPE: LlamaRotaryEmbedding
x	The input tensor.
seq_len	The length of the desired subset. Defaults to None. TYPE: int DEFAULT: None

RETURNS	DESCRIPTION
tuple	A tuple containing two tensors. The first tensor represents the subset of cached cosine values, and the second tensor represents the subset of cached sine values. Both tensors are of the same dtype as x.

RAISES	DESCRIPTION
TypeError	If seq_len is not an integer or None.
ValueError	If seq_len is less than or equal to 0.
AttributeError	If seq_len exceeds the maximum sequence length that has been cached.

Note

The returned subset will include elements up to the index 'seq_len - 1' from the cached cosine and sine values. If seq_len is None or not provided, the entire cached cosine and sine values will be returned.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def forward(self, x, seq_len=None):
    """
    forwards a subset of the cached cosine and sine values based on the given sequence length.

    Args:
        self (LlamaRotaryEmbedding): An instance of the LlamaRotaryEmbedding class.
        x: The input tensor.
        seq_len (int, optional): The length of the desired subset. Defaults to None.

    Returns:
        tuple: A tuple containing two tensors. The first tensor represents the subset of cached cosine values,
            and the second tensor represents the subset of cached sine values. Both tensors are of the
            same dtype as x.

    Raises:
        TypeError: If seq_len is not an integer or None.
        ValueError: If seq_len is less than or equal to 0.
        AttributeError: If seq_len exceeds the maximum sequence length that has been cached.

    Note:
        The returned subset will include elements up to the index 'seq_len - 1' from the cached cosine and sine values.
        If seq_len is None or not provided, the entire cached cosine and sine values will be returned.
    """
    # x: [bs, num_attention_heads, seq_len, head_size]
    if seq_len > self.max_seq_len_cached:
        self._set_cos_sin_cache(seq_len=seq_len, dtype=x.dtype)

    return (
        # 1. default tensor slice
        # self.cos_cached[:seq_len].to(dtype=x.dtype),
        # self.sin_cached[:seq_len].to(dtype=x.dtype),
        # 2. use stridedslice
        # ops.getitem(self.cos_cached, slice(None, seq_len, None)).to(dtype=x.dtype),
        # ops.getitem(self.sin_cached, slice(None, seq_len, None)).to(dtype=x.dtype),
        # 3. use pyboost narrow
        ops.narrow(self.cos_cached, 0, 0, seq_len).to(dtype=x.dtype),
        ops.narrow(self.sin_cached, 0, 0, seq_len).to(dtype=x.dtype),
    )

mindnlp.transformers.models.llama.modeling_llama.apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1)

Applies Rotary Position Embedding to the query and key tensors.

PARAMETER	DESCRIPTION
q	The query tensor. TYPE: `mindspore.Tensor`
k	The key tensor. TYPE: `mindspore.Tensor`
cos	The cosine part of the rotary embedding. TYPE: `mindspore.Tensor`
sin	The sine part of the rotary embedding. TYPE: `mindspore.Tensor`
position_ids	The position indices of the tokens corresponding to the query and key tensors. For example, this can be used to pass offsetted position ids when working with a KV-cache. TYPE: `mindspore.Tensor`
unsqueeze_dim	The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1

RETURNS	DESCRIPTION
	tuple(mindspore.Tensor) comprising of the query and key tensors rotated using the Rotary Position Embedding.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
    """
    Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`mindspore.Tensor`): The query tensor.
        k (`mindspore.Tensor`): The key tensor.
        cos (`mindspore.Tensor`): The cosine part of the rotary embedding.
        sin (`mindspore.Tensor`): The sine part of the rotary embedding.
        position_ids (`mindspore.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
        `tuple(mindspore.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    """
    # cos = cos[position_ids].unsqueeze(unsqueeze_dim)
    # sin = sin[position_ids].unsqueeze(unsqueeze_dim)
    cos = F.embedding(position_ids, cos).unsqueeze(unsqueeze_dim)
    sin = F.embedding(position_ids, sin).unsqueeze(unsqueeze_dim)
    q_embed = ops.add(ops.mul(q, cos), ops.mul(rotate_half(q), sin))
    k_embed = ops.add(ops.mul(k, cos), ops.mul(rotate_half(k), sin))
    return q_embed, k_embed

mindnlp.transformers.models.llama.modeling_llama.repeat_kv(hidden_states, n_rep)

This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def repeat_kv(hidden_states: mindspore.Tensor, n_rep: int) -> mindspore.Tensor:
    """
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    """
    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
    if n_rep == 1:
        return hidden_states
    hidden_states = ops.broadcast_to(hidden_states[:, :, None, :, :], (batch, num_key_value_heads, n_rep, slen, head_dim))
    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)

mindnlp.transformers.models.llama.modeling_llama.rotate_half(x)

Rotates half the hidden dims of the input.

Source code in mindnlp/transformers/models/llama/modeling_llama.py

def rotate_half(x):
    """Rotates half the hidden dims of the input."""
    # 0. use default tensor slice
    # x1 = x[..., : x.shape[-1] // 2]
    # x2 = x[..., x.shape[-1] // 2 :]
    # 1. use tensor_split
    # (x1, x2) = x.tensor_split(2, axis=-1)
    # 2. use pyboost split
    x1, x2 = ops.split(x, x.shape[-1] // 2, dim=-1)
    return ops.cat((-x2, x1), dim=x.ndim-1)

mindnlp.transformers.models.llama.configuration_llama

LLaMA model configuration

mindnlp.transformers.models.llama.configuration_llama.LlamaConfig

Bases: PretrainedConfig

This is the configuration class to store the configuration of a [LlamaModel]. It is used to instantiate an LLaMA model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LLaMA-7B.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
vocab_size	Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the inputs_ids passed when calling [LlamaModel] TYPE: `int`, *optional*, defaults to 32000 DEFAULT: 32000
hidden_size	Dimension of the hidden representations. TYPE: `int`, *optional*, defaults to 4096 DEFAULT: 4096
intermediate_size	Dimension of the MLP representations. TYPE: `int`, *optional*, defaults to 11008 DEFAULT: 11008
num_hidden_layers	Number of hidden layers in the Transformer decoder. TYPE: `int`, *optional*, defaults to 32 DEFAULT: 32
num_attention_heads	Number of attention heads for each attention layer in the Transformer decoder. TYPE: `int`, *optional*, defaults to 32 DEFAULT: 32
num_key_value_heads	This is the number of key_value heads that should be used to implement Grouped Query Attention. If num_key_value_heads=num_attention_heads, the model will use Multi Head Attention (MHA), if num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be forwarded by meanpooling all the original heads within that group. For more details checkout this paper. If it is not specified, will default to num_attention_heads. TYPE: `int`, *optional* DEFAULT: None
hidden_act	The non-linear activation function (function or string) in the decoder. TYPE: `str` or `function`, *optional*, defaults to `"silu"` DEFAULT: 'silu'
max_position_embeddings	The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens, Llama 2 up to 4096, CodeLlama up to 16384. TYPE: `int`, *optional*, defaults to 2048 DEFAULT: 2048
initializer_range	The standard deviation of the truncated_normal_initializer for initializing all weight matrices. TYPE: `float`, *optional*, defaults to 0.02 DEFAULT: 0.02
rms_norm_eps	The epsilon used by the rms normalization layers. TYPE: `float`, *optional*, defaults to 1e-06 DEFAULT: 1e-06
use_cache	Whether or not the model should return the last key/values attentions (not used by all models). Only relevant if config.is_decoder=True. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
pad_token_id	Padding token id. TYPE: `int`, *optional* DEFAULT: None
bos_token_id	Beginning of stream token id. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
eos_token_id	End of stream token id. TYPE: `int`, *optional*, defaults to 2 DEFAULT: 2
pretraining_tp	Experimental feature. Tensor parallelism rank used during pretraining. Please refer to this document to understand more about it. This value is necessary to ensure exact reproducibility of the pretraining results. Please refer to this issue. TYPE: `int`, *optional*, defaults to 1 DEFAULT: 1
tie_word_embeddings	Whether to tie weight embeddings TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
rope_theta	The base period of the RoPE embeddings. TYPE: `float`, *optional*, defaults to 10000.0 DEFAULT: 10000.0
rope_scaling	Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is {"type": strategy name, "factor": scaling factor}. When using this flag, don't update max_position_embeddings to the expected new maximum. See the following thread for more information on how these scaling strategies behave: https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an experimental feature, subject to breaking API changes in future versions. TYPE: `Dict`, *optional* DEFAULT: None
attention_bias	Whether to use a bias in the query, key, value and output projection layers during self-attention. TYPE: `bool`, defaults to `False`, *optional*, defaults to `False` DEFAULT: False
attention_dropout	The dropout ratio for the attention probabilities. TYPE: `float`, *optional*, defaults to 0.0 DEFAULT: 0.0

Example

>>> from transformers import LlamaModel, LlamaConfig
...
>>> # Initializing a LLaMA llama-7b style configuration
>>> configuration = LlamaConfig()
...
>>> # Initializing a model from the llama-7b style configuration
>>> model = LlamaModel(configuration)
...
>>> # Accessing the model configuration
>>> configuration = model.config

Source code in mindnlp/transformers/models/llama/configuration_llama.py

class LlamaConfig(PretrainedConfig):
    r"""
    This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the LLaMA-7B.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 32000):
            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`LlamaModel`]
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 11008):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be forwarded
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens,
            Llama 2 up to 4096, CodeLlama up to 16384.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 1):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 2):
            End of stream token id.
        pretraining_tp (`int`, *optional*, defaults to 1):
            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
            document](https://hf-mirror.com/docs/transformers/parallelism) to understand more about it. This value is
            necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
            issue](https://github.com/pytorch/pytorch/issues/76232).
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
            `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
            these scaling strategies behave:
            https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
            experimental feature, subject to breaking API changes in future versions.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.

    Example:
        ```python
        >>> from transformers import LlamaModel, LlamaConfig
        ...
        >>> # Initializing a LLaMA llama-7b style configuration
        >>> configuration = LlamaConfig()
        ...
        >>> # Initializing a model from the llama-7b style configuration
        >>> model = LlamaModel(configuration)
        ...
        >>> # Accessing the model configuration
        >>> configuration = model.config
        ```
    """
    model_type = "llama"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=32000,
        hidden_size=4096,
        intermediate_size=11008,
        num_hidden_layers=32,
        num_attention_heads=32,
        num_key_value_heads=None,
        hidden_act="silu",
        max_position_embeddings=2048,
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        use_cache=True,
        pad_token_id=None,
        bos_token_id=1,
        eos_token_id=2,
        pretraining_tp=1,
        tie_word_embeddings=False,
        rope_theta=10000.0,
        rope_scaling=None,
        attention_bias=False,
        attention_dropout=0.0,
        **kwargs,
    ):
        """
        This method initializes an instance of the LlamaConfig class.

        Args:
            vocab_size (int, optional): The size of the vocabulary. Default is 32000.
            hidden_size (int, optional): The size of the hidden layers. Default is 4096.
            intermediate_size (int, optional): The size of the intermediate layers. Default is 11008.
            num_hidden_layers (int, optional): The number of hidden layers. Default is 32.
            num_attention_heads (int, optional): The number of attention heads. Default is 32.
            num_key_value_heads (int, optional): The number of key and value heads. If not provided, it defaults to num_attention_heads.
            hidden_act (str, optional): The activation function for the hidden layers. Default is 'silu'.
            max_position_embeddings (int, optional): The maximum position embeddings. Default is 2048.
            initializer_range (float, optional): The range for weight initialization. Default is 0.02.
            rms_norm_eps (float, optional): The epsilon value for RMS normalization. Default is 1e-06.
            pretraining_tp (int, optional): The pretraining TP value. Default is 1.
            use_cache (bool, optional): Indicates whether to use cache. Default is True.
            pad_token_id (int, optional): The ID of the padding token.
            bos_token_id (int, optional): The ID of the beginning of sequence token. Default is 1.
            eos_token_id (int, optional): The ID of the end of sequence token. Default is 2.
            tie_word_embeddings (bool, optional): Indicates whether to tie word embeddings. Default is False.
            rope_theta (float, optional): The theta value for ROPE. Default is 10000.0.
            rope_scaling (None or float, optional): The scaling value for ROPE. If provided, it should be validated.
            attention_bias (bool, optional): Indicates whether to use attention bias. Default is False.
            attention_dropout (float, optional): The dropout rate for attention. Default is 0.0.

        Returns:
            None.

        Raises:
            ValueError: If rope_scaling is provided and it does not pass the validation.
        """
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads

        # for backward compatibility
        if num_key_value_heads is None:
            num_key_value_heads = num_attention_heads

        self.num_key_value_heads = num_key_value_heads
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.rms_norm_eps = rms_norm_eps
        self.pretraining_tp = pretraining_tp
        self.use_cache = use_cache
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self._rope_scaling_validation()
        self.attention_bias = attention_bias
        self.attention_dropout = attention_dropout

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )

    def _rope_scaling_validation(self):
        """
        Validate the `rope_scaling` configuration.
        """
        if self.rope_scaling is None:
            return

        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
            raise ValueError(
                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
                f"got {self.rope_scaling}"
            )
        rope_scaling_type = self.rope_scaling.get("type", None)
        rope_scaling_factor = self.rope_scaling.get("factor", None)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
            )
        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
            raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")

mindnlp.transformers.models.llama.configuration_llama.LlamaConfig.__init__(vocab_size=32000, hidden_size=4096, intermediate_size=11008, num_hidden_layers=32, num_attention_heads=32, num_key_value_heads=None, hidden_act='silu', max_position_embeddings=2048, initializer_range=0.02, rms_norm_eps=1e-06, use_cache=True, pad_token_id=None, bos_token_id=1, eos_token_id=2, pretraining_tp=1, tie_word_embeddings=False, rope_theta=10000.0, rope_scaling=None, attention_bias=False, attention_dropout=0.0, **kwargs)

This method initializes an instance of the LlamaConfig class.

PARAMETER	DESCRIPTION
vocab_size	The size of the vocabulary. Default is 32000. TYPE: int DEFAULT: 32000
hidden_size	The size of the hidden layers. Default is 4096. TYPE: int DEFAULT: 4096
intermediate_size	The size of the intermediate layers. Default is 11008. TYPE: int DEFAULT: 11008
num_hidden_layers	The number of hidden layers. Default is 32. TYPE: int DEFAULT: 32
num_attention_heads	The number of attention heads. Default is 32. TYPE: int DEFAULT: 32
num_key_value_heads	The number of key and value heads. If not provided, it defaults to num_attention_heads. TYPE: int DEFAULT: None
hidden_act	The activation function for the hidden layers. Default is 'silu'. TYPE: str DEFAULT: 'silu'
max_position_embeddings	The maximum position embeddings. Default is 2048. TYPE: int DEFAULT: 2048
initializer_range	The range for weight initialization. Default is 0.02. TYPE: float DEFAULT: 0.02
rms_norm_eps	The epsilon value for RMS normalization. Default is 1e-06. TYPE: float DEFAULT: 1e-06
pretraining_tp	The pretraining TP value. Default is 1. TYPE: int DEFAULT: 1
use_cache	Indicates whether to use cache. Default is True. TYPE: bool DEFAULT: True
pad_token_id	The ID of the padding token. TYPE: int DEFAULT: None
bos_token_id	The ID of the beginning of sequence token. Default is 1. TYPE: int DEFAULT: 1
eos_token_id	The ID of the end of sequence token. Default is 2. TYPE: int DEFAULT: 2
tie_word_embeddings	Indicates whether to tie word embeddings. Default is False. TYPE: bool DEFAULT: False
rope_theta	The theta value for ROPE. Default is 10000.0. TYPE: float DEFAULT: 10000.0
rope_scaling	The scaling value for ROPE. If provided, it should be validated. TYPE: None or float DEFAULT: None
attention_bias	Indicates whether to use attention bias. Default is False. TYPE: bool DEFAULT: False
attention_dropout	The dropout rate for attention. Default is 0.0. TYPE: float DEFAULT: 0.0

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If rope_scaling is provided and it does not pass the validation.

Source code in mindnlp/transformers/models/llama/configuration_llama.py

def __init__(
    self,
    vocab_size=32000,
    hidden_size=4096,
    intermediate_size=11008,
    num_hidden_layers=32,
    num_attention_heads=32,
    num_key_value_heads=None,
    hidden_act="silu",
    max_position_embeddings=2048,
    initializer_range=0.02,
    rms_norm_eps=1e-6,
    use_cache=True,
    pad_token_id=None,
    bos_token_id=1,
    eos_token_id=2,
    pretraining_tp=1,
    tie_word_embeddings=False,
    rope_theta=10000.0,
    rope_scaling=None,
    attention_bias=False,
    attention_dropout=0.0,
    **kwargs,
):
    """
    This method initializes an instance of the LlamaConfig class.

    Args:
        vocab_size (int, optional): The size of the vocabulary. Default is 32000.
        hidden_size (int, optional): The size of the hidden layers. Default is 4096.
        intermediate_size (int, optional): The size of the intermediate layers. Default is 11008.
        num_hidden_layers (int, optional): The number of hidden layers. Default is 32.
        num_attention_heads (int, optional): The number of attention heads. Default is 32.
        num_key_value_heads (int, optional): The number of key and value heads. If not provided, it defaults to num_attention_heads.
        hidden_act (str, optional): The activation function for the hidden layers. Default is 'silu'.
        max_position_embeddings (int, optional): The maximum position embeddings. Default is 2048.
        initializer_range (float, optional): The range for weight initialization. Default is 0.02.
        rms_norm_eps (float, optional): The epsilon value for RMS normalization. Default is 1e-06.
        pretraining_tp (int, optional): The pretraining TP value. Default is 1.
        use_cache (bool, optional): Indicates whether to use cache. Default is True.
        pad_token_id (int, optional): The ID of the padding token.
        bos_token_id (int, optional): The ID of the beginning of sequence token. Default is 1.
        eos_token_id (int, optional): The ID of the end of sequence token. Default is 2.
        tie_word_embeddings (bool, optional): Indicates whether to tie word embeddings. Default is False.
        rope_theta (float, optional): The theta value for ROPE. Default is 10000.0.
        rope_scaling (None or float, optional): The scaling value for ROPE. If provided, it should be validated.
        attention_bias (bool, optional): Indicates whether to use attention bias. Default is False.
        attention_dropout (float, optional): The dropout rate for attention. Default is 0.0.

    Returns:
        None.

    Raises:
        ValueError: If rope_scaling is provided and it does not pass the validation.
    """
    self.vocab_size = vocab_size
    self.max_position_embeddings = max_position_embeddings
    self.hidden_size = hidden_size
    self.intermediate_size = intermediate_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads

    # for backward compatibility
    if num_key_value_heads is None:
        num_key_value_heads = num_attention_heads

    self.num_key_value_heads = num_key_value_heads
    self.hidden_act = hidden_act
    self.initializer_range = initializer_range
    self.rms_norm_eps = rms_norm_eps
    self.pretraining_tp = pretraining_tp
    self.use_cache = use_cache
    self.rope_theta = rope_theta
    self.rope_scaling = rope_scaling
    self._rope_scaling_validation()
    self.attention_bias = attention_bias
    self.attention_dropout = attention_dropout

    super().__init__(
        pad_token_id=pad_token_id,
        bos_token_id=bos_token_id,
        eos_token_id=eos_token_id,
        tie_word_embeddings=tie_word_embeddings,
        **kwargs,
    )

mindnlp.transformers.models.llama.tokenization_llama

Tokenization classes for LLaMA.

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer

Bases: PreTrainedTokenizer

Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as there is no padding token in the original model.

PARAMETER	DESCRIPTION
vocab_file	Path to the vocabulary file. TYPE: `str`
unk_token	The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. TYPE: `str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"` DEFAULT: '<unk>'
bos_token	The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. TYPE: `str` or `tokenizers.AddedToken`, *optional*, defaults to `"<s>"` DEFAULT: '<s>'
eos_token	The end of sequence token. TYPE: `str` or `tokenizers.AddedToken`, *optional*, defaults to `"</s>"` DEFAULT: '</s>'
pad_token	A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by attention mechanisms or loss computation. TYPE: `str` or `tokenizers.AddedToken`, *optional* DEFAULT: None
sp_model_kwargs	Will be passed to the SentencePieceProcessor.__init__() method. The Python wrapper for SentencePiece can be used, among other things, to set: enable_sampling: Enable subword regularization. nbest_size: Sampling parameters for unigram. Invalid for BPE-Dropout. nbest_size = {0,1}: No sampling is performed. nbest_size > 1: samples from the nbest_size results. nbest_size < 0: assuming that nbest_size is infinite and samples from the all hypothesis (lattice) using forward-filtering-and-backward-sampling algorithm. alpha: Smoothing parameter for unigram sampling, and dropout probability of merge operations for BPE-dropout. TYPE: `Dict[str, Any]`, `Optional`, *optional* DEFAULT: None
add_bos_token	Whether or not to add an bos_token at the start of sequences. TYPE: `bool`, *optional*, defaults to `True` DEFAULT: True
add_eos_token	Whether or not to add an eos_token at the end of sequences. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
clean_up_tokenization_spaces	Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra spaces. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
use_default_system_prompt	Whether or not the default system prompt for Llama should be used. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
spaces_between_special_tokens	Whether or not to add spaces between special tokens. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
legacy	Whether or not the legacy behavior of the tokenizer should be used. Legacy is before the merge of #24622 and #25224 which includes fixes to properly handle tokens that appear after special tokens. A simple example: - legacy=True: >>> from transformers import T5Tokenizer ... >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=True) >>> tokenizer.encode("Hello <extra_id_0>.") [8774, 32099, 3, 5, 1] - legacy=False: >>> from transformers import T5Tokenizer ... >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False) >>> tokenizer.encode("Hello <extra_id_0>.") # the extra space `[3]` is no longer here [8774, 32099, 5, 1] Checkout the pull request for more details. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

class LlamaTokenizer(PreTrainedTokenizer):
    """
    Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as there is
    no padding token in the original model.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
        eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        pad_token (`str` or `tokenizers.AddedToken`, *optional*):
            A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by
            attention mechanisms or loss computation.
        sp_model_kwargs (`Dict[str, Any]`, `Optional`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:

            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.

                - `nbest_size = {0,1}`: No sampling is performed.
                - `nbest_size > 1`: samples from the nbest_size results.
                - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.

            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.

        add_bos_token (`bool`, *optional*, defaults to `True`):
            Whether or not to add an `bos_token` at the start of sequences.
        add_eos_token (`bool`, *optional*, defaults to `False`):
            Whether or not to add an `eos_token` at the end of sequences.
        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
            Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like
            extra spaces.
        use_default_system_prompt (`bool`, *optional*, defaults to `False`):
            Whether or not the default system prompt for Llama should be used.
        spaces_between_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not to add spaces between special tokens.
        legacy (`bool`, *optional*):
            Whether or not the `legacy` behavior of the tokenizer should be used. Legacy is before the merge of #24622
            and #25224 which includes fixes to properly handle tokens that appear after special tokens.

            A simple example:
                - `legacy=True`:
                ```python
                >>> from transformers import T5Tokenizer
                ...
                >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=True)
                >>> tokenizer.encode("Hello <extra_id_0>.")
                [8774, 32099, 3, 5, 1]
                ```
                - `legacy=False`:
                ```python
                >>> from transformers import T5Tokenizer
                ...
                >>> tokenizer = T5Tokenizer.from_pretrained("t5-base", legacy=False)
                >>> tokenizer.encode("Hello <extra_id_0>.")  # the extra space `[3]` is no longer here
                [8774, 32099, 5, 1]
                ```
            Checkout the [pull request](https://github.com/huggingface/transformers/pull/24565) for more details.

    """
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        unk_token="<unk>",
        bos_token="<s>",
        eos_token="</s>",
        pad_token=None,
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        add_bos_token=True,
        add_eos_token=False,
        clean_up_tokenization_spaces=False,
        use_default_system_prompt=False,
        spaces_between_special_tokens=False,
        legacy=None,
        **kwargs,
    ):
        """
        Initializes a new instance of the LlamaTokenizer class.

        Args:
            self: The instance of the class.
            vocab_file (str): The path to the vocabulary file.
            unk_token (str, optional): The unknown token. Defaults to '<unk>'.
            bos_token (str, optional): The beginning of sentence token. Defaults to '<s>'.
            eos_token (str, optional): The end of sentence token. Defaults to '</s>'.
            pad_token (str, optional): The padding token. Defaults to None.
            sp_model_kwargs (Dict[str, Any], optional): Additional arguments for the sentencepiece model. Defaults to None.
            add_bos_token (bool, optional): Whether to add the beginning of sentence token. Defaults to True.
            add_eos_token (bool, optional): Whether to add the end of sentence token. Defaults to False.
            clean_up_tokenization_spaces (bool, optional): Whether to clean up tokenization spaces. Defaults to False.
            use_default_system_prompt (bool, optional): Whether to use the default system prompt. Defaults to False.
            spaces_between_special_tokens (bool, optional): Whether to add spaces between special tokens. Defaults to False.
            legacy (bool, optional): Whether to use the legacy behavior. Defaults to None.

        Returns:
            None.

        Raises:
            None.

        Note:
            You are using the default legacy behavior of the LlamaTokenizer. This means that the previous behavior
            will be used, and nothing changes. If you want to use the new behavior, set `legacy=False`.
            Only set this if you understand the implications and have thoroughly read the reason for this change
            as explained in https://github.com/huggingface/transformers/pull/24565.
        """
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
        bos_token = AddedToken(bos_token, normalized=False, special=True) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, normalized=False, special=True) if isinstance(eos_token, str) else eos_token
        unk_token = AddedToken(unk_token, normalized=False, special=True) if isinstance(unk_token, str) else unk_token
        pad_token = AddedToken(pad_token, normalized=False, special=True) if isinstance(pad_token, str) else pad_token

        if legacy is None:
            logger.warning_once(
                f"You are using the default legacy behaviour of the {self.__class__}. This is"
                " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
                " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
                " means, and thoroughly read the reason why this was added as explained in"
                " https://github.com/huggingface/transformers/pull/24565"
            )
            legacy = True

        self.legacy = legacy
        self.vocab_file = vocab_file
        self.add_bos_token = add_bos_token
        self.add_eos_token = add_eos_token
        self.use_default_system_prompt = use_default_system_prompt
        self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            add_bos_token=add_bos_token,
            add_eos_token=add_eos_token,
            sp_model_kwargs=self.sp_model_kwargs,
            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
            use_default_system_prompt=use_default_system_prompt,
            spaces_between_special_tokens=spaces_between_special_tokens,
            legacy=legacy,
            **kwargs,
        )

    @property
    def unk_token_length(self):
        """
        Returns the length of the unknown token in the LlamaTokenizer.

        Args:
            self: An instance of the LlamaTokenizer class.

        Returns:
            int: The method returns the length of the unknown token as an integer value.

        Raises:
            None.

        This method calculates and returns the length of the unknown token in the LlamaTokenizer.
        The unknown token is represented as a string and is encoded using the sp_model.encode() method.
        The length of the encoded unknown token is then determined using the len() function and returned as
        an integer value. The method does not modify any internal state or variables of the LlamaTokenizer class.

        Example:
            ```python
            >>> tokenizer = LlamaTokenizer()
            >>> unk_token_length = tokenizer.unk_token_length()
            >>> print(unk_token_length)  # Output: 5
            ```
        """
        return len(self.sp_model.encode(str(self.unk_token)))

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor
    def get_spm_processor(self, from_slow=False):
        """
        Retrieves the SentencePieceProcessor instance for the LlamaTokenizer.

        Args:
            self (LlamaTokenizer): The instance of LlamaTokenizer.
            from_slow (bool): A flag indicating whether to load the tokenizer from a slow source. Defaults to False.

        Returns:
            None.

        Raises:
            None.
        """
        tokenizer = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        if self.legacy or from_slow:  # no dependency on protobuf
            tokenizer.Load(self.vocab_file)
            return tokenizer

        with open(self.vocab_file, "rb") as f:
            sp_model = f.read()
            model_pb2 = import_protobuf(f"The new behaviour of {self.__class__.__name__} (with `self.legacy = False`)")
            model = model_pb2.ModelProto.FromString(sp_model)
            normalizer_spec = model_pb2.NormalizerSpec()
            normalizer_spec.add_dummy_prefix = False
            model.normalizer_spec.MergeFrom(normalizer_spec)
            sp_model = model.SerializeToString()
            tokenizer.LoadFromSerializedProto(sp_model)
        return tokenizer

    def __getstate__(self):
        """
        Method to serialize the state of the LlamaTokenizer instance for pickling.

        Args:
            self (LlamaTokenizer): The instance of the LlamaTokenizer class.
                Represents the current instance of the tokenizer.

        Returns:
            None: This method does not explicitly return a value, but it updates the state of the tokenizer object.
                The state dictionary contains a copy of the instance's attributes with modifications as needed for
                serialization.

        Raises:
            None
        """
        state = self.__dict__.copy()
        state["sp_model"] = None
        state["sp_model_proto"] = self.sp_model.serialized_model_proto()
        return state

    def __setstate__(self, d):
        """
        This method '__setstate__' in the class 'LlamaTokenizer' is responsible for restoring the state of the object
        from a dictionary representation.

        Args:
            self (object): The instance of the class.
            d (dict): A dictionary containing the state information to be restored.
                It should include the necessary data to reforward the object's state.

        Returns:
            None: The method does not explicitly return any value,
                as it operates by directly updating the object's state.

        Raises:
            TypeError: If the provided 'd' parameter is not a dictionary.
            AttributeError: If the necessary attributes are not present in the dictionary 'd'.
            ValueError: If there are issues with loading or reforwarding the 'sp_model' using the provided data.
        """
        self.__dict__ = d
        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

    @property
    def vocab_size(self):
        """Returns vocab size"""
        return self.sp_model.get_piece_size()

    def get_vocab(self):
        """Returns vocab as a dict"""
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.tokenize
    def tokenize(self, text: "TextInput", add_special_tokens=False, **kwargs) -> List[str]:
        """
        Converts a string to a list of tokens. If `self.legacy` is set to `False`, a prefix token is added unless the
        first token is special.
        """
        if self.legacy or len(text) == 0:
            return super().tokenize(text, **kwargs)

        tokens = super().tokenize(SPIECE_UNDERLINE + text.replace(SPIECE_UNDERLINE, " "), **kwargs)

        if len(tokens) > 1 and tokens[0] == SPIECE_UNDERLINE and tokens[1] in self.all_special_tokens:
            tokens = tokens[1:]
        return tokens

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer._tokenize
    def _tokenize(self, text, **kwargs):
        """
        Returns a tokenized string.

        We de-activated the `add_dummy_prefix` option, thus the sentencepiece internals will always strip any
        SPIECE_UNDERLINE. For example: `self.sp_model.encode(f"{SPIECE_UNDERLINE}Hey", out_type = str)` will give
        `['H', 'e', 'y']` instead of `['▁He', 'y']`. Thus we always encode `f"{unk_token}text"` and strip the
        `unk_token`. Here is an example with `unk_token = "<unk>"` and `unk_token_length = 4`.
        `self.tokenizer.sp_model.encode("<unk> Hey", out_type = str)[4:]`.
        """
        tokens = self.sp_model.encode(text, out_type=str)
        if self.legacy or not text.startswith((SPIECE_UNDERLINE, " ")):
            return tokens

        # 1. Encode string + prefix ex: "<unk> Hey"
        tokens = self.sp_model.encode(self.unk_token + text, out_type=str)
        # 2. Remove self.unk_token from ['<','unk','>', '▁Hey']
        return tokens[self.unk_token_length :] if len(tokens) >= self.unk_token_length else tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""
        token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        # since we manually add the prefix space, we have to remove it when decoding
        if tokens[0].startswith(SPIECE_UNDERLINE):
            tokens[0] = tokens[0][1:]

        current_sub_tokens = []
        out_string = ""
        prev_is_special = False
        for i, token in enumerate(tokens):
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                if not prev_is_special and i != 0 and self.legacy:
                    out_string += " "
                out_string += self.sp_model.decode(current_sub_tokens) + token
                prev_is_special = True
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
                prev_is_special = False
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string

    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
        """
        Save the vocabulary and special tokens file to a directory.

        Args:
            save_directory (`str`):
                The directory in which to save the vocabulary.

        Returns:
            `Tuple(str)`: Paths to the files saved.
        """
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        '''
        This method builds inputs with special tokens for a LlamaTokenizer.

        Args:
            self: The instance of the LlamaTokenizer class.
            token_ids_0: A list of token IDs representing the first sequence.
            token_ids_1 (optional): A list of token IDs representing the second sequence.
                Defaults to None if not provided.

        Returns:
            A list of token IDs with special tokens added at the beginning and end of the sequences.

        Raises:
            None
        '''
        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
        eos_token_id = [self.eos_token_id] if self.add_eos_token else []

        output = bos_token_id + token_ids_0 + eos_token_id

        if token_ids_1 is not None:
            output = output + bos_token_id + token_ids_1 + eos_token_id

        return output

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        """
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        """
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        bos_token_id = [1] if self.add_bos_token else []
        eos_token_id = [1] if self.add_eos_token else []

        if token_ids_1 is None:
            return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
        return (
            bos_token_id
            + ([0] * len(token_ids_0))
            + eos_token_id
            + bos_token_id
            + ([0] * len(token_ids_1))
            + eos_token_id
        )

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        """
        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
        sequence pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        if token_ids_1 is None, only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of ids.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        """
        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
        eos_token_id = [self.eos_token_id] if self.add_eos_token else []

        output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)

        if token_ids_1 is not None:
            output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)

        return output

    @property
    def default_chat_template(self):
        """
        LLaMA uses [INST] and [/INST] to indicate user messages, and <<SYS>> and <</SYS>> to indicate system messages.
        Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict
        user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering
        rather than needing special tokens. The system message is partly 'embedded' in the first user message, which
        results in an unusual token ordering when it is present. This template should definitely be changed if you wish
        to fine-tune a model with more flexible role ordering!

        The output should look something like:

        <bos>[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer <eos><bos>[INST] Prompt [/INST] Answer <eos>
        <bos>[INST] Prompt [/INST]

        The reference for this chat template is [this code
        snippet](https://github.com/facebookresearch/llama/blob/556949fdfb72da27c2f4a40b7f0e4cf0b8153a28/llama/generation.py#L320-L362)
        in the original repository.
        """
        logger.warning_once(
            "\nNo chat template is defined for this tokenizer - using the default template "
            f"for the {self.__class__.__name__} class. If the default is not appropriate for "
            "your model, please set `tokenizer.chat_template` to an appropriate template. "
            "See https://hf-mirror.com/docs/transformers/main/chat_templating for more information.\n"
        )
        template = (
            "{% if messages[0]['role'] == 'system' %}"
            "{% set loop_messages = messages[1:] %}"  # Extract system message if it's present
            "{% set system_message = messages[0]['content'] %}"
            "{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}"
            "{% set loop_messages = messages %}"  # Or use the default system message if the flag is set
            "{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}"
            "{% else %}"
            "{% set loop_messages = messages %}"
            "{% set system_message = false %}"
            "{% endif %}"
            "{% for message in loop_messages %}"  # Loop over all non-system messages
            "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}"
            "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}"
            "{% endif %}"
            "{% if loop.index0 == 0 and system_message != false %}"  # Embed system message in first message
            "{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}"
            "{% else %}"
            "{% set content = message['content'] %}"
            "{% endif %}"
            "{% if message['role'] == 'user' %}"  # After all of that, handle messages/roles in a fairly normal way
            "{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}"
            "{% elif message['role'] == 'system' %}"
            "{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}"
            "{% elif message['role'] == 'assistant' %}"
            "{{ ' '  + content.strip() + ' ' + eos_token }}"
            "{% endif %}"
            "{% endfor %}"
        )
        template = template.replace("USE_DEFAULT_PROMPT", "true" if self.use_default_system_prompt else "false")
        default_message = DEFAULT_SYSTEM_PROMPT.replace("\n", "\\n").replace("'", "\\'")
        template = template.replace("DEFAULT_SYSTEM_MESSAGE", default_message)

        return template

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.default_chat_template property

LLaMA uses [INST] and [/INST] to indicate user messages, and <> and <> to indicate system messages. Assistant messages do not have special tokens, because LLaMA chat models are generally trained with strict user/assistant/user/assistant message ordering, and so assistant messages can be identified from the ordering rather than needing special tokens. The system message is partly 'embedded' in the first user message, which results in an unusual token ordering when it is present. This template should definitely be changed if you wish to fine-tune a model with more flexible role ordering!

The output should look something like:

[INST] B_SYS SystemPrompt E_SYS Prompt [/INST] Answer [INST] Prompt [/INST] Answer [INST] Prompt [/INST]

The reference for this chat template is this code snippet in the original repository.

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.unk_token_length property

Returns the length of the unknown token in the LlamaTokenizer.

PARAMETER	DESCRIPTION
self	An instance of the LlamaTokenizer class.

RETURNS	DESCRIPTION
int	The method returns the length of the unknown token as an integer value.

This method calculates and returns the length of the unknown token in the LlamaTokenizer. The unknown token is represented as a string and is encoded using the sp_model.encode() method. The length of the encoded unknown token is then determined using the len() function and returned as an integer value. The method does not modify any internal state or variables of the LlamaTokenizer class.

Example

>>> tokenizer = LlamaTokenizer()
>>> unk_token_length = tokenizer.unk_token_length()
>>> print(unk_token_length)  # Output: 5

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.vocab_size property

Returns vocab size

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.__getstate__()

Method to serialize the state of the LlamaTokenizer instance for pickling.

PARAMETER	DESCRIPTION
self	The instance of the LlamaTokenizer class. Represents the current instance of the tokenizer. TYPE: LlamaTokenizer

RETURNS	DESCRIPTION
None	This method does not explicitly return a value, but it updates the state of the tokenizer object. The state dictionary contains a copy of the instance's attributes with modifications as needed for serialization.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def __getstate__(self):
    """
    Method to serialize the state of the LlamaTokenizer instance for pickling.

    Args:
        self (LlamaTokenizer): The instance of the LlamaTokenizer class.
            Represents the current instance of the tokenizer.

    Returns:
        None: This method does not explicitly return a value, but it updates the state of the tokenizer object.
            The state dictionary contains a copy of the instance's attributes with modifications as needed for
            serialization.

    Raises:
        None
    """
    state = self.__dict__.copy()
    state["sp_model"] = None
    state["sp_model_proto"] = self.sp_model.serialized_model_proto()
    return state

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.__init__(vocab_file, unk_token='<unk>', bos_token='<s>', eos_token='</s>', pad_token=None, sp_model_kwargs=None, add_bos_token=True, add_eos_token=False, clean_up_tokenization_spaces=False, use_default_system_prompt=False, spaces_between_special_tokens=False, legacy=None, **kwargs)

Initializes a new instance of the LlamaTokenizer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
vocab_file	The path to the vocabulary file. TYPE: str
unk_token	The unknown token. Defaults to ''. TYPE: str DEFAULT: '<unk>'
bos_token	The beginning of sentence token. Defaults to ''. TYPE: str DEFAULT: '<s>'
eos_token	The end of sentence token. Defaults to ''. TYPE: str DEFAULT: '</s>'
pad_token	The padding token. Defaults to None. TYPE: str DEFAULT: None
sp_model_kwargs	Additional arguments for the sentencepiece model. Defaults to None. TYPE: Dict[str, Any] DEFAULT: None
add_bos_token	Whether to add the beginning of sentence token. Defaults to True. TYPE: bool DEFAULT: True
add_eos_token	Whether to add the end of sentence token. Defaults to False. TYPE: bool DEFAULT: False
clean_up_tokenization_spaces	Whether to clean up tokenization spaces. Defaults to False. TYPE: bool DEFAULT: False
use_default_system_prompt	Whether to use the default system prompt. Defaults to False. TYPE: bool DEFAULT: False
spaces_between_special_tokens	Whether to add spaces between special tokens. Defaults to False. TYPE: bool DEFAULT: False
legacy	Whether to use the legacy behavior. Defaults to None. TYPE: bool DEFAULT: None

RETURNS	DESCRIPTION
	None.

Note

You are using the default legacy behavior of the LlamaTokenizer. This means that the previous behavior will be used, and nothing changes. If you want to use the new behavior, set legacy=False. Only set this if you understand the implications and have thoroughly read the reason for this change as explained in https://github.com/huggingface/transformers/pull/24565.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def __init__(
    self,
    vocab_file,
    unk_token="<unk>",
    bos_token="<s>",
    eos_token="</s>",
    pad_token=None,
    sp_model_kwargs: Optional[Dict[str, Any]] = None,
    add_bos_token=True,
    add_eos_token=False,
    clean_up_tokenization_spaces=False,
    use_default_system_prompt=False,
    spaces_between_special_tokens=False,
    legacy=None,
    **kwargs,
):
    """
    Initializes a new instance of the LlamaTokenizer class.

    Args:
        self: The instance of the class.
        vocab_file (str): The path to the vocabulary file.
        unk_token (str, optional): The unknown token. Defaults to '<unk>'.
        bos_token (str, optional): The beginning of sentence token. Defaults to '<s>'.
        eos_token (str, optional): The end of sentence token. Defaults to '</s>'.
        pad_token (str, optional): The padding token. Defaults to None.
        sp_model_kwargs (Dict[str, Any], optional): Additional arguments for the sentencepiece model. Defaults to None.
        add_bos_token (bool, optional): Whether to add the beginning of sentence token. Defaults to True.
        add_eos_token (bool, optional): Whether to add the end of sentence token. Defaults to False.
        clean_up_tokenization_spaces (bool, optional): Whether to clean up tokenization spaces. Defaults to False.
        use_default_system_prompt (bool, optional): Whether to use the default system prompt. Defaults to False.
        spaces_between_special_tokens (bool, optional): Whether to add spaces between special tokens. Defaults to False.
        legacy (bool, optional): Whether to use the legacy behavior. Defaults to None.

    Returns:
        None.

    Raises:
        None.

    Note:
        You are using the default legacy behavior of the LlamaTokenizer. This means that the previous behavior
        will be used, and nothing changes. If you want to use the new behavior, set `legacy=False`.
        Only set this if you understand the implications and have thoroughly read the reason for this change
        as explained in https://github.com/huggingface/transformers/pull/24565.
    """
    self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
    bos_token = AddedToken(bos_token, normalized=False, special=True) if isinstance(bos_token, str) else bos_token
    eos_token = AddedToken(eos_token, normalized=False, special=True) if isinstance(eos_token, str) else eos_token
    unk_token = AddedToken(unk_token, normalized=False, special=True) if isinstance(unk_token, str) else unk_token
    pad_token = AddedToken(pad_token, normalized=False, special=True) if isinstance(pad_token, str) else pad_token

    if legacy is None:
        logger.warning_once(
            f"You are using the default legacy behaviour of the {self.__class__}. This is"
            " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
            " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
            " means, and thoroughly read the reason why this was added as explained in"
            " https://github.com/huggingface/transformers/pull/24565"
        )
        legacy = True

    self.legacy = legacy
    self.vocab_file = vocab_file
    self.add_bos_token = add_bos_token
    self.add_eos_token = add_eos_token
    self.use_default_system_prompt = use_default_system_prompt
    self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

    super().__init__(
        bos_token=bos_token,
        eos_token=eos_token,
        unk_token=unk_token,
        pad_token=pad_token,
        add_bos_token=add_bos_token,
        add_eos_token=add_eos_token,
        sp_model_kwargs=self.sp_model_kwargs,
        clean_up_tokenization_spaces=clean_up_tokenization_spaces,
        use_default_system_prompt=use_default_system_prompt,
        spaces_between_special_tokens=spaces_between_special_tokens,
        legacy=legacy,
        **kwargs,
    )

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.__setstate__(d)

This method 'setstate' in the class 'LlamaTokenizer' is responsible for restoring the state of the object from a dictionary representation.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
d	A dictionary containing the state information to be restored. It should include the necessary data to reforward the object's state. TYPE: dict

RETURNS	DESCRIPTION
None	The method does not explicitly return any value, as it operates by directly updating the object's state.

RAISES	DESCRIPTION
TypeError	If the provided 'd' parameter is not a dictionary.
AttributeError	If the necessary attributes are not present in the dictionary 'd'.
ValueError	If there are issues with loading or reforwarding the 'sp_model' using the provided data.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def __setstate__(self, d):
    """
    This method '__setstate__' in the class 'LlamaTokenizer' is responsible for restoring the state of the object
    from a dictionary representation.

    Args:
        self (object): The instance of the class.
        d (dict): A dictionary containing the state information to be restored.
            It should include the necessary data to reforward the object's state.

    Returns:
        None: The method does not explicitly return any value,
            as it operates by directly updating the object's state.

    Raises:
        TypeError: If the provided 'd' parameter is not a dictionary.
        AttributeError: If the necessary attributes are not present in the dictionary 'd'.
        ValueError: If there are issues with loading or reforwarding the 'sp_model' using the provided data.
    """
    self.__dict__ = d
    self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.build_inputs_with_special_tokens(token_ids_0, token_ids_1=None)

This method builds inputs with special tokens for a LlamaTokenizer.

PARAMETER	DESCRIPTION
self	The instance of the LlamaTokenizer class.
token_ids_0	A list of token IDs representing the first sequence.
token_ids_1	A list of token IDs representing the second sequence. Defaults to None if not provided. TYPE: optional DEFAULT: None

RETURNS	DESCRIPTION
	A list of token IDs with special tokens added at the beginning and end of the sequences.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
    '''
    This method builds inputs with special tokens for a LlamaTokenizer.

    Args:
        self: The instance of the LlamaTokenizer class.
        token_ids_0: A list of token IDs representing the first sequence.
        token_ids_1 (optional): A list of token IDs representing the second sequence.
            Defaults to None if not provided.

    Returns:
        A list of token IDs with special tokens added at the beginning and end of the sequences.

    Raises:
        None
    '''
    bos_token_id = [self.bos_token_id] if self.add_bos_token else []
    eos_token_id = [self.eos_token_id] if self.add_eos_token else []

    output = bos_token_id + token_ids_0 + eos_token_id

    if token_ids_1 is not None:
        output = output + bos_token_id + token_ids_1 + eos_token_id

    return output

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.convert_tokens_to_string(tokens)

Converts a sequence of tokens (string) in a single string.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def convert_tokens_to_string(self, tokens):
    """Converts a sequence of tokens (string) in a single string."""
    # since we manually add the prefix space, we have to remove it when decoding
    if tokens[0].startswith(SPIECE_UNDERLINE):
        tokens[0] = tokens[0][1:]

    current_sub_tokens = []
    out_string = ""
    prev_is_special = False
    for i, token in enumerate(tokens):
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
            if not prev_is_special and i != 0 and self.legacy:
                out_string += " "
            out_string += self.sp_model.decode(current_sub_tokens) + token
            prev_is_special = True
            current_sub_tokens = []
        else:
            current_sub_tokens.append(token)
            prev_is_special = False
    out_string += self.sp_model.decode(current_sub_tokens)
    return out_string

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.create_token_type_ids_from_sequences(token_ids_0, token_ids_1=None)

Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT sequence pair mask has the following format:

0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
| first sequence    | second sequence |

if token_ids_1 is None, only returns the first portion of the mask (0s).

PARAMETER	DESCRIPTION
token_ids_0	List of ids. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
List[int]	List[int]: List of token type IDs according to the given sequence(s).

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def create_token_type_ids_from_sequences(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
    """
    Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
    sequence pair mask has the following format:

    ```
    0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
    | first sequence    | second sequence |
    ```

    if token_ids_1 is None, only returns the first portion of the mask (0s).

    Args:
        token_ids_0 (`List[int]`):
            List of ids.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.

    Returns:
        `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
    """
    bos_token_id = [self.bos_token_id] if self.add_bos_token else []
    eos_token_id = [self.eos_token_id] if self.add_eos_token else []

    output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)

    if token_ids_1 is not None:
        output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)

    return output

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.get_special_tokens_mask(token_ids_0, token_ids_1=None, already_has_special_tokens=False)

Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer prepare_for_model method.

PARAMETER	DESCRIPTION
token_ids_0	List of IDs. TYPE: `List[int]`
token_ids_1	Optional second list of IDs for sequence pairs. TYPE: `List[int]`, *optional* DEFAULT: None
already_has_special_tokens	Whether or not the token list is already formatted with special tokens for the model. TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False

RETURNS	DESCRIPTION
List[int]	List[int]: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.

Source code in mindnlp/transformers/models/llama/tokenization_llama.py

def get_special_tokens_mask(
    self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
) -> List[int]:
    """
    Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
    special tokens using the tokenizer `prepare_for_model` method.

    Args:
        token_ids_0 (`List[int]`):
            List of IDs.
        token_ids_1 (`List[int]`, *optional*):
            Optional second list of IDs for sequence pairs.
        already_has_special_tokens (`bool`, *optional*, defaults to `False`):
            Whether or not the token list is already formatted with special tokens for the model.

    Returns:
        `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
    """
    if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
        )

    bos_token_id = [1] if self.add_bos_token else []
    eos_token_id = [1] if self.add_eos_token else []

    if token_ids_1 is None:
        return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
    return (
        bos_token_id
        + ([0] * len(token_ids_0))
        + eos_token_id
        + bos_token_id
        + ([0] * len(token_ids_1))
        + eos_token_id
    )

mindnlp.transformers.models.llama.tokenization_llama.LlamaTokenizer.get_spm_processor(from_slow=False)

Retrieves the SentencePieceProcessor instance for the LlamaTokenizer.