whisper

mindnlp.transformers.models.whisper.modeling_whisper

MindSpore Whisper model.

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""
    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[WhisperConfig] = None,
    ):
        """
        Initializes the WhisperAttention class.

        Args:
            self: The instance of the class.
            embed_dim (int): The dimension of the input embeddings.
            num_heads (int): The number of attention heads.
            dropout (float, optional): The dropout probability. Default is 0.0.
            is_decoder (bool, optional): Indicates whether the attention mechanism is used as a decoder.
                Default is False.
            bias (bool, optional): Indicates whether the linear layers have bias terms. Default is True.
            is_causal (bool, optional): Indicates whether the attention is causal. Default is False.
            config (Optional[WhisperConfig], optional): The configuration for WhisperAttention. Default is None.

        Returns:
            None.

        Raises:
            ValueError: If the embed_dim is not divisible by num_heads.
        """
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    # Copied from transformers.models.bart.modeling_bart.BartAttention._shape with BART->whisper
    def _shape(self, tensor: mindspore.Tensor, seq_len: int, bsz: int):
        """
        Reshapes the input tensor for attention computation.

        Args:
            self (WhisperAttention): An instance of the WhisperAttention class.
            tensor (mindspore.Tensor): The input tensor to be reshaped.
                It should have shape (bsz * seq_len, self.embed_dim).
            seq_len (int): The length of the sequence.
            bsz (int): The batch size.

        Returns:
            None.

        Raises:
            None.
        """
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).swapaxes(1, 2)

    # Copied from transformers.models.bart.modeling_bart.BartAttention.forward with BART->whisper
    def forward(
        self,
        hidden_states: mindspore.Tensor,
        key_value_states: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        layer_head_mask: Optional[mindspore.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
        """Input shape: Batch x Time x Channel"""
        # if key_value_states are provided this layer is used as a cross-attention layer
        # for the decoder
        is_cross_attention = key_value_states is not None

        bsz, tgt_len, _ = hidden_states.shape

        # get query proj
        query_states = self.q_proj(hidden_states) * self.scaling
        # get key, value proj
        # `past_key_value[0].shape[2] == key_value_states.shape[1]`
        # is checking that the `sequence_length` of the `past_key_value` is the same as
        # the provided `key_value_states` to support prefix tuning
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == key_value_states.shape[1]
        ):
            # reuse k,v, cross_attentions
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            # cross_attentions
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            # reuse k, v, self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = ops.cat([past_key_value[0], key_states], axis=2)
            value_states = ops.cat([past_key_value[1], value_states], axis=2)
        else:
            # self_attention
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
        if self.is_decoder:
            # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.shape[1]
        attn_weights = ops.bmm(query_states, key_states.swapaxes(1, 2))

        if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.shape}"
            )

        if attention_mask is not None:
            if attention_mask.shape != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = ops.softmax(attn_weights, axis=-1)

        if layer_head_mask is not None:
            if layer_head_mask.shape != (self.num_heads,):
                raise ValueError(
                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                    f" {layer_head_mask.shape}"
                )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        if output_attentions:
            # this operation is a bit awkward, but it's required to
            # make sure that attn_weights keeps its gradient.
            # In order to do so, attn_weights have to be reshaped
            # twice and have to be reused in the following
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = ops.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = ops.bmm(attn_probs, value_states)
        if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.shape}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.swapaxes(1, 2)

        # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
        # partitioned across GPUs when using tensor-parallelism.
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention.__init__(embed_dim, num_heads, dropout=0.0, is_decoder=False, bias=True, is_causal=False, config=None)

Initializes the WhisperAttention class.

PARAMETER	DESCRIPTION
self	The instance of the class.
embed_dim	The dimension of the input embeddings. TYPE: int
num_heads	The number of attention heads. TYPE: int
dropout	The dropout probability. Default is 0.0. TYPE: float DEFAULT: 0.0
is_decoder	Indicates whether the attention mechanism is used as a decoder. Default is False. TYPE: bool DEFAULT: False
bias	Indicates whether the linear layers have bias terms. Default is True. TYPE: bool DEFAULT: True
is_causal	Indicates whether the attention is causal. Default is False. TYPE: bool DEFAULT: False
config	The configuration for WhisperAttention. Default is None. TYPE: Optional[WhisperConfig] DEFAULT: None

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the embed_dim is not divisible by num_heads.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(
    self,
    embed_dim: int,
    num_heads: int,
    dropout: float = 0.0,
    is_decoder: bool = False,
    bias: bool = True,
    is_causal: bool = False,
    config: Optional[WhisperConfig] = None,
):
    """
    Initializes the WhisperAttention class.

    Args:
        self: The instance of the class.
        embed_dim (int): The dimension of the input embeddings.
        num_heads (int): The number of attention heads.
        dropout (float, optional): The dropout probability. Default is 0.0.
        is_decoder (bool, optional): Indicates whether the attention mechanism is used as a decoder.
            Default is False.
        bias (bool, optional): Indicates whether the linear layers have bias terms. Default is True.
        is_causal (bool, optional): Indicates whether the attention is causal. Default is False.
        config (Optional[WhisperConfig], optional): The configuration for WhisperAttention. Default is None.

    Returns:
        None.

    Raises:
        ValueError: If the embed_dim is not divisible by num_heads.
    """
    super().__init__()
    self.embed_dim = embed_dim
    self.num_heads = num_heads
    self.dropout = dropout
    self.head_dim = embed_dim // num_heads
    self.config = config

    if (self.head_dim * num_heads) != self.embed_dim:
        raise ValueError(
            f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
            f" and `num_heads`: {num_heads})."
        )
    self.scaling = self.head_dim**-0.5
    self.is_decoder = is_decoder
    self.is_causal = is_causal

    self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False)
    self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
    self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
    self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperAttention.forward(hidden_states, key_value_states=None, past_key_value=None, attention_mask=None, layer_head_mask=None, output_attentions=False)

Input shape: Batch x Time x Channel

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    key_value_states: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    layer_head_mask: Optional[mindspore.Tensor] = None,
    output_attentions: bool = False,
) -> Tuple[mindspore.Tensor, Optional[mindspore.Tensor], Optional[Tuple[mindspore.Tensor]]]:
    """Input shape: Batch x Time x Channel"""
    # if key_value_states are provided this layer is used as a cross-attention layer
    # for the decoder
    is_cross_attention = key_value_states is not None

    bsz, tgt_len, _ = hidden_states.shape

    # get query proj
    query_states = self.q_proj(hidden_states) * self.scaling
    # get key, value proj
    # `past_key_value[0].shape[2] == key_value_states.shape[1]`
    # is checking that the `sequence_length` of the `past_key_value` is the same as
    # the provided `key_value_states` to support prefix tuning
    if (
        is_cross_attention
        and past_key_value is not None
        and past_key_value[0].shape[2] == key_value_states.shape[1]
    ):
        # reuse k,v, cross_attentions
        key_states = past_key_value[0]
        value_states = past_key_value[1]
    elif is_cross_attention:
        # cross_attentions
        key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
        value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
    elif past_key_value is not None:
        # reuse k, v, self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
        key_states = ops.cat([past_key_value[0], key_states], axis=2)
        value_states = ops.cat([past_key_value[1], value_states], axis=2)
    else:
        # self_attention
        key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
        value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
    if self.is_decoder:
        # if cross_attention save Tuple(mindspore.Tensor, mindspore.Tensor) of all cross attention key/value_states.
        # Further calls to cross_attention layer can then reuse all cross-attention
        # key/value_states (first "if" case)
        # if uni-directional self-attention (decoder) save Tuple(mindspore.Tensor, mindspore.Tensor) of
        # all previous decoder key/value_states. Further calls to uni-directional self-attention
        # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
        # if encoder bi-directional self-attention `past_key_value` is always `None`
        past_key_value = (key_states, value_states)

    proj_shape = (bsz * self.num_heads, -1, self.head_dim)
    query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
    key_states = key_states.reshape(*proj_shape)
    value_states = value_states.reshape(*proj_shape)

    src_len = key_states.shape[1]
    attn_weights = ops.bmm(query_states, key_states.swapaxes(1, 2))

    if attn_weights.shape != (bsz * self.num_heads, tgt_len, src_len):
        raise ValueError(
            f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
            f" {attn_weights.shape}"
        )

    if attention_mask is not None:
        if attention_mask.shape != (bsz, 1, tgt_len, src_len):
            raise ValueError(
                f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.shape}"
            )
        attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
        attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

    attn_weights = ops.softmax(attn_weights, axis=-1)

    if layer_head_mask is not None:
        if layer_head_mask.shape != (self.num_heads,):
            raise ValueError(
                f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                f" {layer_head_mask.shape}"
            )
        attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
        attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

    if output_attentions:
        # this operation is a bit awkward, but it's required to
        # make sure that attn_weights keeps its gradient.
        # In order to do so, attn_weights have to be reshaped
        # twice and have to be reused in the following
        attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
        attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
    else:
        attn_weights_reshaped = None

    attn_probs = ops.dropout(attn_weights, p=self.dropout, training=self.training)

    attn_output = ops.bmm(attn_probs, value_states)
    if attn_output.shape != (bsz * self.num_heads, tgt_len, self.head_dim):
        raise ValueError(
            f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
            f" {attn_output.shape}"
        )

    attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
    attn_output = attn_output.swapaxes(1, 2)

    # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
    # partitioned across GPUs when using tensor-parallelism.
    attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
    attn_output = self.out_proj(attn_output)

    return attn_output, attn_weights_reshaped, past_key_value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder

Bases: WhisperPreTrainedModel

Transformer decoder consisting of config.decoder_layers layers. Each layer is a [WhisperDecoderLayer]

PARAMETER	DESCRIPTION
config	WhisperConfig TYPE: WhisperConfig

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoder(WhisperPreTrainedModel):
    """
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`WhisperDecoderLayer`]

    Args:
        config: WhisperConfig
    """
    main_input_name = "input_ids"

    def __init__(self, config: WhisperConfig):
        """
        Initializes the WhisperDecoder class.

        Args:
            self: The instance of the class.
            config (WhisperConfig):
                An instance of WhisperConfig containing the configuration parameters for the decoder.

                - dropout (float): The dropout probability.
                - decoder_layerdrop (float): The layer dropout probability for the decoder.
                - pad_token_id (int): The token id used for padding.
                - max_target_positions (int): The maximum target sequence length.
                - max_source_positions (int): The maximum source sequence length.
                - d_model (int): The dimensionality of the model.
                - scale_embedding (bool): Indicates whether to scale the embeddings.
                - vocab_size (int): The size of the vocabulary.
                - decoder_layers (int): The number of decoder layers.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_target_positions = config.max_target_positions
        self.max_source_positions = config.max_source_positions
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

        self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=self.padding_idx)
        self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model)

        self.layers = nn.ModuleList([WhisperDecoderLayer(config) for _ in range(config.decoder_layers)])

        self.layer_norm = nn.LayerNorm([config.d_model])

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        """
        Returns the input embeddings for the WhisperDecoder class.

        Args:
            self: An instance of the WhisperDecoder class.

        Returns:
            embed_tokens: This method returns the input embeddings for the decoder.

        Raises:
            None.
        """
        return self.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperDecoder class.

        Args:
            self (WhisperDecoder): The instance of the WhisperDecoder class.
            value: The input embeddings to be set for the WhisperDecoder.
                This parameter should be of the appropriate type and format required for input embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.embed_tokens = value

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        encoder_hidden_states=None,
        head_mask=None,
        cross_attn_head_mask=None,
        past_key_values=None,
        inputs_embeds=None,
        use_cache=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it.

                Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                of the decoder.
            head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

            cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention
                on hidden heads. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

                If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
                that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
                all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of
                shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
                `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
                control over how to convert `input_ids` indices into associated vectors than the model's internal
                embedding lookup matrix.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # retrieve input_ids and inputs_embeds
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        if input_ids is not None:
            input_shape = input_ids.shape
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.shape[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        # past_key_values_length
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids)

        if getattr(self.config, "_flash_attn_2_enabled", False):
            # 2d mask is passed through the layers
            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
        else:
            # 4d mask is passed through the layers
            attention_mask = _prepare_4d_causal_attention_mask(
                attention_mask, input_shape, inputs_embeds, past_key_values_length
            )

        # embed positions
        if input_ids is not None:
            positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length)
        else:
            positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length)

        hidden_states = inputs_embeds + positions
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..."
                )
                use_cache = False
        # decoder layers
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
        for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
            if attn_mask is not None:
                assert attn_mask.shape[0] == (len(self.layers)), (
                    f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                    f" {head_mask.shape[0]}."
                )
        for idx, decoder_layer in enumerate(self.layers):
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = ops.rand((1,))
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None
            layer_outputs = decoder_layer(
                hidden_states,
                attention_mask=attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                cross_attn_layer_head_mask=(
                    cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
                ),
                past_key_value=past_key_value,
                output_attentions=output_attentions,
                use_cache=use_cache,
            )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[2],)

        hidden_states = self.layer_norm(hidden_states)
        # add hidden states from the last decoder layer
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.__init__(config)

Initializes the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	An instance of WhisperConfig containing the configuration parameters for the decoder. dropout (float): The dropout probability. decoder_layerdrop (float): The layer dropout probability for the decoder. pad_token_id (int): The token id used for padding. max_target_positions (int): The maximum target sequence length. max_source_positions (int): The maximum source sequence length. d_model (int): The dimensionality of the model. scale_embedding (bool): Indicates whether to scale the embeddings. vocab_size (int): The size of the vocabulary. decoder_layers (int): The number of decoder layers. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes the WhisperDecoder class.

    Args:
        self: The instance of the class.
        config (WhisperConfig):
            An instance of WhisperConfig containing the configuration parameters for the decoder.

            - dropout (float): The dropout probability.
            - decoder_layerdrop (float): The layer dropout probability for the decoder.
            - pad_token_id (int): The token id used for padding.
            - max_target_positions (int): The maximum target sequence length.
            - max_source_positions (int): The maximum source sequence length.
            - d_model (int): The dimensionality of the model.
            - scale_embedding (bool): Indicates whether to scale the embeddings.
            - vocab_size (int): The size of the vocabulary.
            - decoder_layers (int): The number of decoder layers.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    self.dropout = config.dropout
    self.layerdrop = config.decoder_layerdrop
    self.padding_idx = config.pad_token_id
    self.max_target_positions = config.max_target_positions
    self.max_source_positions = config.max_source_positions
    self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

    self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, padding_idx=self.padding_idx)
    self.embed_positions = WhisperPositionalEmbedding(self.max_target_positions, config.d_model)

    self.layers = nn.ModuleList([WhisperDecoderLayer(config) for _ in range(config.decoder_layers)])

    self.layer_norm = nn.LayerNorm([config.d_model])

    self.gradient_checkpointing = False
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.forward(input_ids=None, attention_mask=None, encoder_hidden_states=None, head_mask=None, cross_attn_head_mask=None, past_key_values=None, inputs_embeds=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [WhisperTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_hidden_states	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. TYPE: `mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
cross_attn_head_mask	Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention on hidden heads. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
past_key_values	Tuple of tuple(mindspore.Tensor) of length config.n_layers, with each tuple having 2 tensors of shape (batch_size, num_heads, sequence_length, embed_size_per_head)) and 2 additional tensors of shape (batch_size, num_heads, encoder_sequence_length, embed_size_per_head). Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see past_key_values input) to speed up sequential decoding. If past_key_values are used, the user can optionally input only the last decoder_input_ids (those that don't have their past key value states given to this model) of shape (batch_size, 1) instead of all decoder_input_ids of shape (batch_size, sequence_length). TYPE: `tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    input_ids=None,
    attention_mask=None,
    encoder_hidden_states=None,
    head_mask=None,
    cross_attn_head_mask=None,
    past_key_values=None,
    inputs_embeds=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
):
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it.

            Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_hidden_states (`mindspore.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention
            on hidden heads. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
            cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
            that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
            all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of
            shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
            `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
            control over how to convert `input_ids` indices into associated vectors than the model's internal
            embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    use_cache = use_cache if use_cache is not None else self.config.use_cache
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # retrieve input_ids and inputs_embeds
    if input_ids is not None and inputs_embeds is not None:
        raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
    if input_ids is not None:
        input_shape = input_ids.shape
        input_ids = input_ids.view(-1, input_shape[-1])
    elif inputs_embeds is not None:
        input_shape = inputs_embeds.shape[:-1]
    else:
        raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

    # past_key_values_length
    past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

    if inputs_embeds is None:
        inputs_embeds = self.embed_tokens(input_ids)

    if getattr(self.config, "_flash_attn_2_enabled", False):
        # 2d mask is passed through the layers
        attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
    else:
        # 4d mask is passed through the layers
        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

    # embed positions
    if input_ids is not None:
        positions = self.embed_positions(input_ids, past_key_values_length=past_key_values_length)
    else:
        positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length)

    hidden_states = inputs_embeds + positions
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

    if self.gradient_checkpointing and self.training:
        if use_cache:
            logger.warning_once(
                "`use_cache = True` is incompatible with gradient checkpointing. Setting `use_cache = False`..."
            )
            use_cache = False
    # decoder layers
    all_hidden_states = () if output_hidden_states else None
    all_self_attns = () if output_attentions else None
    all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
    next_decoder_cache = () if use_cache else None

    # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired
    for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
        if attn_mask is not None:
            assert attn_mask.shape[0] == (len(self.layers)), (
                f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                f" {head_mask.shape[0]}."
            )
    for idx, decoder_layer in enumerate(self.layers):
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        if output_hidden_states:
            all_hidden_states += (hidden_states,)
        if self.training:
            dropout_probability = ops.rand((1,))
            if dropout_probability < self.layerdrop:
                continue

        past_key_value = past_key_values[idx] if past_key_values is not None else None
        layer_outputs = decoder_layer(
            hidden_states,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            layer_head_mask=(head_mask[idx] if head_mask is not None else None),
            cross_attn_layer_head_mask=(
                cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
            ),
            past_key_value=past_key_value,
            output_attentions=output_attentions,
            use_cache=use_cache,
        )
        hidden_states = layer_outputs[0]

        if use_cache:
            next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

        if output_attentions:
            all_self_attns += (layer_outputs[1],)

            if encoder_hidden_states is not None:
                all_cross_attentions += (layer_outputs[2],)

    hidden_states = self.layer_norm(hidden_states)
    # add hidden states from the last decoder layer
    if output_hidden_states:
        all_hidden_states += (hidden_states,)

    next_cache = next_decoder_cache if use_cache else None
    if not return_dict:
        return tuple(
            v
            for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
            if v is not None
        )
    return BaseModelOutputWithPastAndCrossAttentions(
        last_hidden_state=hidden_states,
        past_key_values=next_cache,
        hidden_states=all_hidden_states,
        attentions=all_self_attns,
        cross_attentions=all_cross_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.get_input_embeddings()

Returns the input embeddings for the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	An instance of the WhisperDecoder class.

RETURNS	DESCRIPTION
embed_tokens	This method returns the input embeddings for the decoder.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self):
    """
    Returns the input embeddings for the WhisperDecoder class.

    Args:
        self: An instance of the WhisperDecoder class.

    Returns:
        embed_tokens: This method returns the input embeddings for the decoder.

    Raises:
        None.
    """
    return self.embed_tokens

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoder.set_input_embeddings(value)

Sets the input embeddings for the WhisperDecoder class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoder class. TYPE: WhisperDecoder
value	The input embeddings to be set for the WhisperDecoder. This parameter should be of the appropriate type and format required for input embeddings.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperDecoder class.

    Args:
        self (WhisperDecoder): The instance of the WhisperDecoder class.
        value: The input embeddings to be set for the WhisperDecoder.
            This parameter should be of the appropriate type and format required for input embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.embed_tokens = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer

Bases: Module

The WhisperDecoderLayer class represents a single layer of the Whisper decoder model, which includes self-attention and cross-attention mechanisms. This class is designed to be used within the WhisperTransformer model for sequence-to-sequence tasks.

This class inherits from nn.Module and contains methods for initializing the layer and performing computations on input tensors. The layer consists of self-attention, encoder attention, feedforward neural network, and layer normalization modules.

The init method sets up the layer with parameters such as embedding dimensions, attention types, dropout rates, activation functions, and normalization layers.

The forward method processes input hidden states through the self-attention mechanism, followed by encoder attention if provided. It also handles dropout, residual connections, and feedforward network transformations. The method allows for caching of key-value states and optionally returns attention weights and cached states.

Please refer to the method docstrings for detailed information on the input and output parameters, as well as their respective shapes and purposes.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoderLayer(nn.Module):

    """
    The WhisperDecoderLayer class represents a single layer of the Whisper decoder model, which includes self-attention
    and cross-attention mechanisms. This class is designed to be used within the WhisperTransformer model for
    sequence-to-sequence tasks.

    This class inherits from nn.Module and contains methods for initializing the layer and performing computations on
    input tensors. The layer consists of self-attention, encoder attention, feedforward neural network, and layer
    normalization modules.

    The __init__ method sets up the layer with parameters such as embedding dimensions, attention types, dropout rates,
    activation functions, and normalization layers.

    The forward method processes input hidden states through the self-attention mechanism, followed by encoder
    attention if provided. It also handles dropout, residual connections, and feedforward network transformations.
    The method allows for caching of key-value states and optionally returns attention weights and cached states.

    Please refer to the method docstrings for detailed information on the input and output parameters, as well as
    their respective shapes and purposes.
    """
    def __init__(self, config: WhisperConfig):
        """
        Initializes a WhisperDecoderLayer object.

        Args:
            self (WhisperDecoderLayer): The current instance of the WhisperDecoderLayer class.
            config (WhisperConfig): An instance of the WhisperConfig class containing configuration settings.

        Returns:
            None.

        Raises:
            ValueError: If the attention type specified in the config is not supported.
            TypeError: If the input parameters are not of the expected types.
            RuntimeError: If an error occurs during the initialization process.
        """
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "default"

        self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            is_causal=True,
            config=config,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout

        self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.encoder_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            config=config,
        )
        self.encoder_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
        self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm([self.embed_dim])

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_hidden_states: Optional[mindspore.Tensor] = None,
        encoder_attention_mask: Optional[mindspore.Tensor] = None,
        layer_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_layer_head_mask: Optional[mindspore.Tensor] = None,
        past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            encoder_hidden_states (`mindspore.Tensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`mindspore.Tensor`): encoder attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            cross_attn_layer_head_mask (`mindspore.Tensor`): mask for cross-attention heads in a given layer of
                size `(decoder_attention_heads,)`.
            past_key_value (`Tuple(mindspore.Tensor)`): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        # Self Attention
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        # add present self-attn cache to positions 1,2 of present_key_value tuple
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        # Cross-Attention Block
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
                output_attentions=output_attentions,
            )

            hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
            hidden_states = residual + hidden_states

            # add cross-attn to positions 3,4 of present_key_value tuple
            present_key_value = present_key_value + cross_attn_present_key_value

        # Fully Connected
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        if use_cache:
            outputs += (present_key_value,)

        return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer.__init__(config)

Initializes a WhisperDecoderLayer object.

PARAMETER	DESCRIPTION
self	The current instance of the WhisperDecoderLayer class. TYPE: WhisperDecoderLayer
config	An instance of the WhisperConfig class containing configuration settings. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
ValueError	If the attention type specified in the config is not supported.
TypeError	If the input parameters are not of the expected types.
RuntimeError	If an error occurs during the initialization process.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes a WhisperDecoderLayer object.

    Args:
        self (WhisperDecoderLayer): The current instance of the WhisperDecoderLayer class.
        config (WhisperConfig): An instance of the WhisperConfig class containing configuration settings.

    Returns:
        None.

    Raises:
        ValueError: If the attention type specified in the config is not supported.
        TypeError: If the input parameters are not of the expected types.
        RuntimeError: If an error occurs during the initialization process.
    """
    super().__init__()
    self.embed_dim = config.d_model
    attn_type = "default"

    self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        embed_dim=self.embed_dim,
        num_heads=config.decoder_attention_heads,
        dropout=config.attention_dropout,
        is_decoder=True,
        is_causal=True,
        config=config,
    )
    self.dropout = config.dropout
    self.activation_fn = ACT2FN[config.activation_function]
    self.activation_dropout = config.activation_dropout

    self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.encoder_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        self.embed_dim,
        config.decoder_attention_heads,
        dropout=config.attention_dropout,
        is_decoder=True,
        config=config,
    )
    self.encoder_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
    self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
    self.final_layer_norm = nn.LayerNorm([self.embed_dim])

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderLayer.forward(hidden_states, attention_mask=None, encoder_hidden_states=None, encoder_attention_mask=None, layer_head_mask=None, cross_attn_layer_head_mask=None, past_key_value=None, output_attentions=False, use_cache=True)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
encoder_hidden_states	cross attention input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor` DEFAULT: None
encoder_attention_mask	encoder attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor` DEFAULT: None
layer_head_mask	mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: `mindspore.Tensor` DEFAULT: None
cross_attn_layer_head_mask	mask for cross-attention heads in a given layer of size (decoder_attention_heads,). TYPE: `mindspore.Tensor` DEFAULT: None
past_key_value	cached past key and value projection states TYPE: `Tuple(mindspore.Tensor)` DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_hidden_states: Optional[mindspore.Tensor] = None,
    encoder_attention_mask: Optional[mindspore.Tensor] = None,
    layer_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_layer_head_mask: Optional[mindspore.Tensor] = None,
    past_key_value: Optional[Tuple[mindspore.Tensor]] = None,
    output_attentions: Optional[bool] = False,
    use_cache: Optional[bool] = True,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`): attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        encoder_hidden_states (`mindspore.Tensor`):
            cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
        encoder_attention_mask (`mindspore.Tensor`): encoder attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        cross_attn_layer_head_mask (`mindspore.Tensor`): mask for cross-attention heads in a given layer of
            size `(decoder_attention_heads,)`.
        past_key_value (`Tuple(mindspore.Tensor)`): cached past key and value projection states
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)

    # Self Attention
    # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
    self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
    # add present self-attn cache to positions 1,2 of present_key_value tuple
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
        hidden_states=hidden_states,
        past_key_value=self_attn_past_key_value,
        attention_mask=attention_mask,
        layer_head_mask=layer_head_mask,
        output_attentions=output_attentions,
    )
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    # Cross-Attention Block
    cross_attn_present_key_value = None
    cross_attn_weights = None
    if encoder_hidden_states is not None:
        residual = hidden_states
        hidden_states = self.encoder_attn_layer_norm(hidden_states)

        # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
            hidden_states=hidden_states,
            key_value_states=encoder_hidden_states,
            attention_mask=encoder_attention_mask,
            layer_head_mask=cross_attn_layer_head_mask,
            past_key_value=cross_attn_past_key_value,
            output_attentions=output_attentions,
        )

        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        # add cross-attn to positions 3,4 of present_key_value tuple
        present_key_value = present_key_value + cross_attn_present_key_value

    # Fully Connected
    residual = hidden_states
    hidden_states = self.final_layer_norm(hidden_states)
    hidden_states = self.activation_fn(self.fc1(hidden_states))
    hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
    hidden_states = self.fc2(hidden_states)
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (self_attn_weights, cross_attn_weights)

    if use_cache:
        outputs += (present_key_value,)

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper

Bases: WhisperPreTrainedModel

This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the [EncoderDecoderModel] framework.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperDecoderWrapper(WhisperPreTrainedModel):
    """
    This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is
    used in combination with the [`EncoderDecoderModel`] framework.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the WhisperDecoderWrapper class.

        Args:
            self: The instance of the class.
            config (object): The configuration object containing the settings for the decoder.
                The config object should have the following attributes:

                - is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder.
                This should be set to False for the WhisperDecoderWrapper class.
                - Other attributes specific to the WhisperDecoder class.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__(config)
        config.is_encoder_decoder = False
        self.decoder = WhisperDecoder(config)

    def get_input_embeddings(self):
        """
        Get input embeddings for the WhisperDecoderWrapper.

        Args:
            self (WhisperDecoderWrapper):
                The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved.

        Returns:
            None.

        Raises:
            None.
        """
        return self.decoder.embed_tokens

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperDecoderWrapper.

        Args:
            self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.
            value (object): The input embeddings to be set for the decoder. It should be an object of the
                desired input embeddings.

        Returns:
            None.

        Raises:
            None.
        """
        self.decoder.embed_tokens = value

    def forward(self, *args, **kwargs):
        """
        Method to forward a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

        Args:
            self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.

        Returns:
            None: This method does not return any value explicitly. It delegates the forwardion to the decoder method.

        Raises:
            None.
        """
        return self.decoder(*args, **kwargs)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.__init__(config)

Initializes a new instance of the WhisperDecoderWrapper class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object containing the settings for the decoder. The config object should have the following attributes: is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder. This should be set to False for the WhisperDecoderWrapper class. Other attributes specific to the WhisperDecoder class. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes a new instance of the WhisperDecoderWrapper class.

    Args:
        self: The instance of the class.
        config (object): The configuration object containing the settings for the decoder.
            The config object should have the following attributes:

            - is_encoder_decoder (bool): Indicates if the WhisperDecoderWrapper is used as an encoder-decoder.
            This should be set to False for the WhisperDecoderWrapper class.
            - Other attributes specific to the WhisperDecoder class.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__(config)
    config.is_encoder_decoder = False
    self.decoder = WhisperDecoder(config)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.forward(*args, **kwargs)

Method to forward a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoderWrapper class. TYPE: WhisperDecoderWrapper

RETURNS	DESCRIPTION
None	This method does not return any value explicitly. It delegates the forwardion to the decoder method.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(self, *args, **kwargs):
    """
    Method to forward a WhisperDecoderWrapper object by invoking the decoder with the provided arguments.

    Args:
        self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.

    Returns:
        None: This method does not return any value explicitly. It delegates the forwardion to the decoder method.

    Raises:
        None.
    """
    return self.decoder(*args, **kwargs)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.get_input_embeddings()

Get input embeddings for the WhisperDecoderWrapper.

PARAMETER	DESCRIPTION
self	The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved. TYPE: WhisperDecoderWrapper

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self):
    """
    Get input embeddings for the WhisperDecoderWrapper.

    Args:
        self (WhisperDecoderWrapper):
            The instance of WhisperDecoderWrapper for which input embeddings are to be retrieved.

    Returns:
        None.

    Raises:
        None.
    """
    return self.decoder.embed_tokens

mindnlp.transformers.models.whisper.modeling_whisper.WhisperDecoderWrapper.set_input_embeddings(value)

Sets the input embeddings for the WhisperDecoderWrapper.

PARAMETER	DESCRIPTION
self	The instance of the WhisperDecoderWrapper class. TYPE: WhisperDecoderWrapper
value	The input embeddings to be set for the decoder. It should be an object of the desired input embeddings. TYPE: object

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperDecoderWrapper.

    Args:
        self (WhisperDecoderWrapper): The instance of the WhisperDecoderWrapper class.
        value (object): The input embeddings to be set for the decoder. It should be an object of the
            desired input embeddings.

    Returns:
        None.

    Raises:
        None.
    """
    self.decoder.embed_tokens = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder

Bases: WhisperPreTrainedModel

Transformer encoder consisting of config.encoder_layers self attention layers. Each layer is a [WhisperEncoderLayer].

PARAMETER	DESCRIPTION
config	WhisperConfig TYPE: WhisperConfig

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperEncoder(WhisperPreTrainedModel):
    """
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
    [`WhisperEncoderLayer`].

    Args:
        config: WhisperConfig
    """
    def __init__(self, config: WhisperConfig):
        """Initialize a WhisperEncoder object.

        Args:
            config (WhisperConfig):
                The configuration object containing the parameters for the encoder.

                - dropout (float): The dropout probability for the encoder.
                - encoder_layerdrop (float): The probability of dropping an entire encoder layer.
                - d_model (int): The embedding dimension size.
                - num_mel_bins (int): The number of mel bins for the input audio.
                - pad_token_id (int): The padding token ID.
                - max_source_positions (int): The maximum number of source positions.
                - scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim).

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop

        embed_dim = config.d_model
        self.num_mel_bins = config.num_mel_bins
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_source_positions
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

        self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1, pad_mode='pad', bias=True)
        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, pad_mode='pad', bias=True)

        self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
        self.embed_positions.weight.requires_grad = False

        self.layers = nn.ModuleList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
        self.layer_norm = nn.LayerNorm([config.d_model])

        self.gradient_checkpointing = False
        # Initialize weights and apply final processing
        self.post_init()

    def _freeze_parameters(self):
        """
        Freeze the parameters of the WhisperEncoder.

        Args:
            self (WhisperEncoder): The instance of WhisperEncoder.

        Returns:
            None.

        Raises:
            None.
        """
        for param in self.get_parameters():
            param.requires_grad = False

    def get_input_embeddings(self) -> nn.Module:
        """
        Get the input embeddings for the WhisperEncoder.

        Args:
            self (WhisperEncoder): The instance of the WhisperEncoder class.

        Returns:
            nn.Module: The input embeddings.

        Raises:
            None.
        """
        return self.conv1

    def set_input_embeddings(self, value: nn.Module):
        """
        Method to set input embeddings for the WhisperEncoder class.

        Args:
            self (WhisperEncoder): The instance of the WhisperEncoder class.
                It is used to access the attributes and methods of the class.
            value (nn.Module): The input embeddings to be set for the WhisperEncoder.
                It should be an instance of the nn.Module class.

        Returns:
            None: This method sets the input embeddings for the WhisperEncoder instance.

        Raises:
            None.
        """
        self.conv1 = value

    def forward(
        self,
        input_features,
        attention_mask=None,
        head_mask=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r"""
        Args:
            input_features (`mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`):
                Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
                `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
                `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding
                and conversion into a tensor of type `mindspore.Tensor`. See [`~WhisperFeatureExtractor.__call__`]
            attention_mask (`mindspore.Tensor`)`, *optional*):
                Whisper does not support masking of the `input_features`, this argument is preserved for compatibility,
                but it is not used. By default the silence in the input log mel spectrogram are ignored.
            head_mask (`mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        inputs_embeds = ops.gelu(self.conv1(input_features))
        inputs_embeds = ops.gelu(self.conv2(inputs_embeds))

        inputs_embeds = inputs_embeds.permute(0, 2, 1)
        embed_pos = self.embed_positions.weight

        hidden_states = inputs_embeds + embed_pos
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        # check if head_mask has a correct number of layers specified if desired
        if head_mask is not None:
            assert head_mask.shape[0] == (
                len(self.layers)
            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.shape[0]}."

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
            to_drop = False
            if self.training:
                dropout_probability = ops.rand((1,))
                if dropout_probability < self.layerdrop:  # skip the layer
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.__call__,
                        hidden_states,
                        None,
                        (head_mask[idx] if head_mask is not None else None),
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        None,
                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.__init__(config)

Initialize a WhisperEncoder object.

PARAMETER

DESCRIPTION

config

The configuration object containing the parameters for the encoder. dropout (float): The dropout probability for the encoder. encoder_layerdrop (float): The probability of dropping an entire encoder layer. d_model (int): The embedding dimension size. num_mel_bins (int): The number of mel bins for the input audio. pad_token_id (int): The padding token ID. max_source_positions (int): The maximum number of source positions. scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim). TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """Initialize a WhisperEncoder object.

    Args:
        config (WhisperConfig):
            The configuration object containing the parameters for the encoder.

            - dropout (float): The dropout probability for the encoder.
            - encoder_layerdrop (float): The probability of dropping an entire encoder layer.
            - d_model (int): The embedding dimension size.
            - num_mel_bins (int): The number of mel bins for the input audio.
            - pad_token_id (int): The padding token ID.
            - max_source_positions (int): The maximum number of source positions.
            - scale_embedding (bool): Whether to scale the embeddings by math.sqrt(embed_dim).

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    self.dropout = config.dropout
    self.layerdrop = config.encoder_layerdrop

    embed_dim = config.d_model
    self.num_mel_bins = config.num_mel_bins
    self.padding_idx = config.pad_token_id
    self.max_source_positions = config.max_source_positions
    self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

    self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1, pad_mode='pad', bias=True)
    self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1, pad_mode='pad', bias=True)

    self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
    self.embed_positions.weight.requires_grad = False

    self.layers = nn.ModuleList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
    self.layer_norm = nn.LayerNorm([config.d_model])

    self.gradient_checkpointing = False
    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.forward(input_features, attention_mask=None, head_mask=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_features	Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a .flac or .wav audio file into an array of type List[float] or a numpy.ndarray, e.g. via the soundfile library (pip install soundfile). To prepare the array into input_features, the [AutoFeatureExtractor] should be used for extracting the mel features, padding and conversion into a tensor of type mindspore.Tensor. See [~WhisperFeatureExtractor.__call__] TYPE: `mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`
attention_mask	Whisper does not support masking of the input_features, this argument is preserved for compatibility, but it is not used. By default the silence in the input log mel spectrogram are ignored. TYPE: `mindspore.Tensor`)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    input_features,
    attention_mask=None,
    head_mask=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
):
    r"""
    Args:
        input_features (`mindspore.Tensor` of shape `(batch_size, feature_size, sequence_length)`):
            Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
            `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
            `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding
            and conversion into a tensor of type `mindspore.Tensor`. See [`~WhisperFeatureExtractor.__call__`]
        attention_mask (`mindspore.Tensor`)`, *optional*):
            Whisper does not support masking of the `input_features`, this argument is preserved for compatibility,
            but it is not used. By default the silence in the input log mel spectrogram are ignored.
        head_mask (`mindspore.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    inputs_embeds = ops.gelu(self.conv1(input_features))
    inputs_embeds = ops.gelu(self.conv2(inputs_embeds))

    inputs_embeds = inputs_embeds.permute(0, 2, 1)
    embed_pos = self.embed_positions.weight

    hidden_states = inputs_embeds + embed_pos
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)

    encoder_states = () if output_hidden_states else None
    all_attentions = () if output_attentions else None

    # check if head_mask has a correct number of layers specified if desired
    if head_mask is not None:
        assert head_mask.shape[0] == (
            len(self.layers)
        ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.shape[0]}."

    for idx, encoder_layer in enumerate(self.layers):
        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)
        # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
        to_drop = False
        if self.training:
            dropout_probability = ops.rand((1,))
            if dropout_probability < self.layerdrop:  # skip the layer
                to_drop = True

        if to_drop:
            layer_outputs = (None, None)
        else:
            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    encoder_layer.__call__,
                    hidden_states,
                    None,
                    (head_mask[idx] if head_mask is not None else None),
                    output_attentions,
                )
            else:
                layer_outputs = encoder_layer(
                    hidden_states,
                    None,
                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

        if output_attentions:
            all_attentions = all_attentions + (layer_outputs[1],)

    hidden_states = self.layer_norm(hidden_states)
    if output_hidden_states:
        encoder_states = encoder_states + (hidden_states,)

    if not return_dict:
        return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
    return BaseModelOutput(
        last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.get_input_embeddings()

Get the input embeddings for the WhisperEncoder.

PARAMETER	DESCRIPTION
self	The instance of the WhisperEncoder class. TYPE: WhisperEncoder

RETURNS	DESCRIPTION
Module	nn.Module: The input embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Module:
    """
    Get the input embeddings for the WhisperEncoder.

    Args:
        self (WhisperEncoder): The instance of the WhisperEncoder class.

    Returns:
        nn.Module: The input embeddings.

    Raises:
        None.
    """
    return self.conv1

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoder.set_input_embeddings(value)

Method to set input embeddings for the WhisperEncoder class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperEncoder class. It is used to access the attributes and methods of the class. TYPE: WhisperEncoder
value	The input embeddings to be set for the WhisperEncoder. It should be an instance of the nn.Module class. TYPE: Module

RETURNS	DESCRIPTION
None	This method sets the input embeddings for the WhisperEncoder instance.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value: nn.Module):
    """
    Method to set input embeddings for the WhisperEncoder class.

    Args:
        self (WhisperEncoder): The instance of the WhisperEncoder class.
            It is used to access the attributes and methods of the class.
        value (nn.Module): The input embeddings to be set for the WhisperEncoder.
            It should be an instance of the nn.Module class.

    Returns:
        None: This method sets the input embeddings for the WhisperEncoder instance.

    Raises:
        None.
    """
    self.conv1 = value

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer

Bases: Module

The WhisperEncoderLayer class represents a single layer of the Whisper Encoder, which is used in the training and inference process of the Whisper model. This class inherits from the nn.Module class.

ATTRIBUTE	DESCRIPTION
`embed_dim`	The dimension size of the input embedding. TYPE: int
`self_attn`	The self-attention module used in the encoder layer. TYPE: Module
`self_attn_layer_norm`	Layer normalization module applied to the output of the self-attention module. TYPE: LayerNorm
`dropout`	Dropout probability applied to the output of the self-attention module. TYPE: float
`activation_fn`	Activation function applied to the output of the feed-forward network. TYPE: function
`activation_dropout`	Dropout probability applied to the output of the activation function. TYPE: float
`fc1`	First fully connected layer of the feed-forward network. TYPE: Linear
`fc2`	Second fully connected layer of the feed-forward network. TYPE: Linear
`final_layer_norm`	Layer normalization module applied to the output of the feed-forward network. TYPE: LayerNorm

METHOD	DESCRIPTION
`forward`	Constructs the encoder layer by applying the self-attention, feed-forward network, and residual connections to the input hidden states.

PARAMETER	DESCRIPTION
`hidden_states`	The input to the layer of shape (batch, seq_len, embed_dim). TYPE: Tensor
`attention_mask`	The attention mask of size (batch, 1, tgt_len, src_len), where padding elements are indicated by very large negative values. TYPE: Tensor
`layer_head_mask`	The mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: Tensor
`output_attentions`	Whether or not to return the attentions tensors of all attention layers. TYPE: bool

RETURNS	DESCRIPTION
	(mindspore.Tensor): The output hidden states of the encoder layer.

Note

The forward method does not include the signatures or any other code.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperEncoderLayer(nn.Module):

    """
    The `WhisperEncoderLayer` class represents a single layer of the Whisper Encoder, which is used in the training
    and inference process of the Whisper model. This class inherits from the `nn.Module` class.

    Attributes:
        `embed_dim` (int): The dimension size of the input embedding.
        `self_attn` (nn.Module): The self-attention module used in the encoder layer.
        `self_attn_layer_norm` (nn.LayerNorm): Layer normalization module applied to the output of the
            self-attention module.
        `dropout` (float): Dropout probability applied to the output of the self-attention module.
        `activation_fn` (function): Activation function applied to the output of the feed-forward network.
        `activation_dropout` (float): Dropout probability applied to the output of the activation function.
        `fc1` (nn.Linear): First fully connected layer of the feed-forward network.
        `fc2` (nn.Linear): Second fully connected layer of the feed-forward network.
        `final_layer_norm` (nn.LayerNorm): Layer normalization module applied to the output of the feed-forward network.

    Methods:
        `forward`: Constructs the encoder layer by applying the self-attention, feed-forward network, and residual
            connections to the input hidden states.

    Args:
        `hidden_states` (mindspore.Tensor): The input to the layer of shape `(batch, seq_len, embed_dim)`.
        `attention_mask` (mindspore.Tensor): The attention mask of size `(batch, 1, tgt_len, src_len)`,
            where padding elements are indicated by very large negative values.
        `layer_head_mask` (mindspore.Tensor): The mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        `output_attentions` (bool, optional): Whether or not to return the attentions tensors of all attention layers.

    Returns:
        `(mindspore.Tensor)`: The output hidden states of the encoder layer.

    Raises:
        None

    Note:
        The forward method does not include the signatures or any other code.
    """
    def __init__(self, config: WhisperConfig):
        """
        Initializes a new instance of the WhisperEncoderLayer class.

        Args:
            self: The instance of the class.
            config (WhisperConfig): The configuration object for the WhisperEncoderLayer.
                It contains various settings and parameters for the WhisperEncoderLayer.

                - config.d_model (int): The embedding dimension.
                - config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2.
                Defaults to False.
                - config.encoder_attention_heads (int): The number of attention heads in the self-attention layer.
                - config.attention_dropout (float): The dropout probability for the attention layer.
                - config.dropout (float): The dropout probability for the layer.
                - config.activation_function (str): The activation function to be used.
                - config.activation_dropout (float): The dropout probability for the activation function.
                - config.encoder_ffn_dim (int): The dimension of the feed-forward network.

        Returns:
            None.

        Raises:
            None.
        """
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

        self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.encoder_attention_heads,
            dropout=config.attention_dropout,
            config=config,
        )
        self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout
        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm([self.embed_dim])

    def forward(
        self,
        hidden_states: mindspore.Tensor,
        attention_mask: mindspore.Tensor,
        layer_head_mask: mindspore.Tensor,
        output_attentions: bool = False,
    ) -> mindspore.Tensor:
        """
        Args:
            hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`mindspore.Tensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        """
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        if hidden_states.dtype == mindspore.float16 and (
            ops.isinf(hidden_states).any() or ops.isnan(hidden_states).any()
        ):
            clamp_value = np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max - 1000
            hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer.__init__(config)

Initializes a new instance of the WhisperEncoderLayer class.

PARAMETER	DESCRIPTION
self	The instance of the class.
config	The configuration object for the WhisperEncoderLayer. It contains various settings and parameters for the WhisperEncoderLayer. config.d_model (int): The embedding dimension. config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2. Defaults to False. config.encoder_attention_heads (int): The number of attention heads in the self-attention layer. config.attention_dropout (float): The dropout probability for the attention layer. config.dropout (float): The dropout probability for the layer. config.activation_function (str): The activation function to be used. config.activation_dropout (float): The dropout probability for the activation function. config.encoder_ffn_dim (int): The dimension of the feed-forward network. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes a new instance of the WhisperEncoderLayer class.

    Args:
        self: The instance of the class.
        config (WhisperConfig): The configuration object for the WhisperEncoderLayer.
            It contains various settings and parameters for the WhisperEncoderLayer.

            - config.d_model (int): The embedding dimension.
            - config._flash_attn_2_enabled (bool, optional): Whether to enable the flash_attention_2.
            Defaults to False.
            - config.encoder_attention_heads (int): The number of attention heads in the self-attention layer.
            - config.attention_dropout (float): The dropout probability for the attention layer.
            - config.dropout (float): The dropout probability for the layer.
            - config.activation_function (str): The activation function to be used.
            - config.activation_dropout (float): The dropout probability for the activation function.
            - config.encoder_ffn_dim (int): The dimension of the feed-forward network.

    Returns:
        None.

    Raises:
        None.
    """
    super().__init__()
    self.embed_dim = config.d_model
    attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

    self.self_attn = WHISPER_ATTENTION_CLASSES[attn_type](
        embed_dim=self.embed_dim,
        num_heads=config.encoder_attention_heads,
        dropout=config.attention_dropout,
        config=config,
    )
    self.self_attn_layer_norm = nn.LayerNorm([self.embed_dim])
    self.dropout = config.dropout
    self.activation_fn = ACT2FN[config.activation_function]
    self.activation_dropout = config.activation_dropout
    self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
    self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
    self.final_layer_norm = nn.LayerNorm([self.embed_dim])

mindnlp.transformers.models.whisper.modeling_whisper.WhisperEncoderLayer.forward(hidden_states, attention_mask, layer_head_mask, output_attentions=False)

PARAMETER	DESCRIPTION
hidden_states	input to the layer of shape (batch, seq_len, embed_dim) TYPE: `mindspore.Tensor`
attention_mask	attention mask of size (batch, 1, tgt_len, src_len) where padding elements are indicated by very large negative values. TYPE: `mindspore.Tensor`
layer_head_mask	mask for attention heads in a given layer of size (encoder_attention_heads,). TYPE: `mindspore.Tensor`
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: False

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    hidden_states: mindspore.Tensor,
    attention_mask: mindspore.Tensor,
    layer_head_mask: mindspore.Tensor,
    output_attentions: bool = False,
) -> mindspore.Tensor:
    """
    Args:
        hidden_states (`mindspore.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
        attention_mask (`mindspore.Tensor`): attention mask of size
            `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
        layer_head_mask (`mindspore.Tensor`): mask for attention heads in a given layer of size
            `(encoder_attention_heads,)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
    """
    residual = hidden_states
    hidden_states = self.self_attn_layer_norm(hidden_states)
    hidden_states, attn_weights, _ = self.self_attn(
        hidden_states=hidden_states,
        attention_mask=attention_mask,
        layer_head_mask=layer_head_mask,
        output_attentions=output_attentions,
    )
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    residual = hidden_states
    hidden_states = self.final_layer_norm(hidden_states)
    hidden_states = self.activation_fn(self.fc1(hidden_states))
    hidden_states = ops.dropout(hidden_states, p=self.activation_dropout, training=self.training)
    hidden_states = self.fc2(hidden_states)
    hidden_states = ops.dropout(hidden_states, p=self.dropout, training=self.training)
    hidden_states = residual + hidden_states

    if hidden_states.dtype == mindspore.float16 and (
        ops.isinf(hidden_states).any() or ops.isnan(hidden_states).any()
    ):
        clamp_value = np.finfo(mindspore.dtype_to_nptype(hidden_states.dtype)).max - 1000
        hidden_states = ops.clamp(hidden_states, min=-clamp_value, max=clamp_value)

    outputs = (hidden_states,)

    if output_attentions:
        outputs += (attn_weights,)

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification

Bases: WhisperPreTrainedModel

This class represents a Whisper model for audio classification tasks. It is a subclass of the WhisperPreTrainedModel class.

The WhisperForAudioClassification class consists of various methods and attributes that are used for audio classification tasks.

METHOD	DESCRIPTION
`__init__`	Initializes the WhisperForAudioClassification instance.
`freeze_encoder`	Disables gradient computation for the Whisper encoder, preventing its parameters from being updated during training.
`get_input_embeddings`	Retrieves the input embeddings from the encoder.
`set_input_embeddings`	Sets the input embeddings for the encoder.
`forward`	Constructs the forward pass of the model for audio classification.

ATTRIBUTE	DESCRIPTION
`encoder`	Instance of the WhisperEncoder class used for encoding audio input.
`layer_weights`	Parameter representing weights for weighted layer sum, if enabled.
`projector`	Instance of the nn.Linear class used for projecting hidden states.
`classifier`	Instance of the nn.Linear class used for classification.
`config`	Configuration object containing model settings.

Example

>>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
>>> from datasets import load_dataset
...
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
>>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
...
>>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
>>> sample = next(iter(ds))
...
>>> inputs = feature_extractor(
...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
... )
>>> input_features = inputs.input_features
...
>>> with torch.no_grad():
>>>     logits = model(input_features).logits
...
>>> predicted_class_ids = torch.argmax(logits).item()
>>> predicted_label = model.config.id2label[predicted_class_ids]
>>> predicted_label
'Afrikaans'

For more details on the class methods and attributes, refer to the individual method docstrings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForAudioClassification(WhisperPreTrainedModel):

    """
    This class represents a Whisper model for audio classification tasks.
    It is a subclass of the `WhisperPreTrainedModel` class.

    The `WhisperForAudioClassification` class consists of various methods and attributes that are used for audio
    classification tasks.

    Methods:
        `__init__`: Initializes the `WhisperForAudioClassification` instance.
        `freeze_encoder`: Disables gradient computation for the Whisper encoder, preventing its parameters
            from being updated during training.
        `get_input_embeddings`: Retrieves the input embeddings from the encoder.
        `set_input_embeddings`: Sets the input embeddings for the encoder.
        `forward`: Constructs the forward pass of the model for audio classification.

    Attributes:
        `encoder`: Instance of the `WhisperEncoder` class used for encoding audio input.
        `layer_weights`: Parameter representing weights for weighted layer sum, if enabled.
        `projector`: Instance of the `nn.Linear` class used for projecting hidden states.
        `classifier`: Instance of the `nn.Linear` class used for classification.
        `config`: Configuration object containing model settings.

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
        >>> from datasets import load_dataset
        ...
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        ...
        >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
        >>> sample = next(iter(ds))
        ...
        >>> inputs = feature_extractor(
        ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
        ... )
        >>> input_features = inputs.input_features
        ...
        >>> with torch.no_grad():
        >>>     logits = model(input_features).logits
        ...
        >>> predicted_class_ids = torch.argmax(logits).item()
        >>> predicted_label = model.config.id2label[predicted_class_ids]
        >>> predicted_label
        'Afrikaans'
        ```

    For more details on the class methods and attributes, refer to the individual method docstrings.
    """
    def __init__(self, config):
        """
        Initializes a new instance of the WhisperForAudioClassification class.

        Args:
            self: The instance of the WhisperForAudioClassification class.
            config: A configuration object containing settings for the model.
                It should be an instance of the configuration class specific to WhisperForAudioClassification.

        Returns:
            None.

        Raises:
            TypeError: If the 'config' parameter is not provided or is not of the expected type.
            ValueError: If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
            RuntimeError: If an error occurs during initialization.
        """
        super().__init__(config)

        self.encoder = WhisperEncoder(config)
        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
        if config.use_weighted_layer_sum:
            self.layer_weights = Parameter(ops.ones(num_layers) / num_layers)
        self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
        self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)

        # Initialize weights and apply final processing
        self.post_init()

    def freeze_encoder(self):
        """
        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
        not be updated during training. Only the projection layers and classification head will be updated.
        """
        self.encoder._freeze_parameters()

    def get_input_embeddings(self) -> nn.Module:
        """
        This method returns the input embeddings from the encoder for audio classification.

        Args:
            self (WhisperForAudioClassification): The instance of the WhisperForAudioClassification class.

        Returns:
            nn.Module: The input embeddings from the encoder for audio classification.

        Raises:
            None
        """
        return self.encoder.get_input_embeddings()

    def set_input_embeddings(self, value: nn.Module):
        """
        Method to set the input embeddings for the WhisperForAudioClassification class.

        Args:
            self:
                The instance of the WhisperForAudioClassification class.

                - Type: WhisperForAudioClassification
                - Purpose: Represents the current instance of the class.
                - Restrictions: None

            value:
                The input embeddings to be set for the encoder.

                - Type: nn.Module
                - Purpose: Represents the input embeddings used for encoding.
                - Restrictions: Should be an instance of nn.Module.

        Returns:
            None.

        Raises:
            None.
        """
        self.encoder.set_input_embeddings(value)

    def forward(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        labels: Optional[mindspore.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
                Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
                config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
                `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:
            Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

        Example:
            ```python
            >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
            >>> from datasets import load_dataset
            ...
            >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
            >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
            ...
            >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
            >>> sample = next(iter(ds))
            ...
            >>> inputs = feature_extractor(
            ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
            ... )
            >>> input_features = inputs.input_features
            ...
            >>> with torch.no_grad():
            ...     logits = model(input_features).logits
            ...
            >>> predicted_class_ids = torch.argmax(logits).item()
            >>> predicted_label = model.config.id2label[predicted_class_ids]
            >>> predicted_label
            'Afrikaans'
            ```
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_features,
                head_mask=head_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        if self.config.use_weighted_layer_sum:
            hidden_states = ops.stack(encoder_outputs, axis=1)
            norm_weights = ops.softmax(self.layer_weights, axis=-1)
            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(axis=1)
        else:
            hidden_states = encoder_outputs[0]

        hidden_states = self.projector(hidden_states)
        pooled_output = hidden_states.mean(axis=1)

        logits = self.classifier(pooled_output)

        loss = None

        if labels is not None:
            loss = ops.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + encoder_outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
        )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.__init__(config)

Initializes a new instance of the WhisperForAudioClassification class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class.
config	A configuration object containing settings for the model. It should be an instance of the configuration class specific to WhisperForAudioClassification.

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the 'config' parameter is not provided or is not of the expected type.
ValueError	If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
RuntimeError	If an error occurs during initialization.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes a new instance of the WhisperForAudioClassification class.

    Args:
        self: The instance of the WhisperForAudioClassification class.
        config: A configuration object containing settings for the model.
            It should be an instance of the configuration class specific to WhisperForAudioClassification.

    Returns:
        None.

    Raises:
        TypeError: If the 'config' parameter is not provided or is not of the expected type.
        ValueError: If the 'num_hidden_layers' attribute in the 'config' parameter is not defined.
        RuntimeError: If an error occurs during initialization.
    """
    super().__init__(config)

    self.encoder = WhisperEncoder(config)
    num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
    if config.use_weighted_layer_sum:
        self.layer_weights = Parameter(ops.ones(num_layers) / num_layers)
    self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
    self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.forward(input_features=None, head_mask=None, encoder_outputs=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the sequence classification/regression loss. Indices should be in [0, ..., config.num_labels - 1]. If config.num_labels == 1 a regression loss is computed (Mean-Square loss), If config.num_labels > 1 a classification loss is computed (Cross-Entropy). TYPE: `mindspore.Tensor` of shape `(batch_size,)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], SequenceClassifierOutput]	Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

Example

>>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
>>> from datasets import load_dataset
...
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
>>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
...
>>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
>>> sample = next(iter(ds))
...
>>> inputs = feature_extractor(
...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
... )
>>> input_features = inputs.input_features
...
>>> with torch.no_grad():
...     logits = model(input_features).logits
...
>>> predicted_class_ids = torch.argmax(logits).item()
>>> predicted_label = model.config.id2label[predicted_class_ids]
>>> predicted_label
'Afrikaans'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    labels: Optional[mindspore.Tensor] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

    Returns:
        Union[Tuple[mindspore.Tensor], SequenceClassifierOutput]

    Example:
        ```python
        >>> from transformers import AutoFeatureExtractor, WhisperForAudioClassification
        >>> from datasets import load_dataset
        ...
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        >>> model = WhisperForAudioClassification.from_pretrained("sanchit-gandhi/whisper-medium-fleurs-lang-id")
        ...
        >>> ds = load_dataset("google/fleurs", "all", split="validation", streaming=True)
        >>> sample = next(iter(ds))
        ...
        >>> inputs = feature_extractor(
        ...     sample["audio"]["array"], sampling_rate=sample["audio"]["sampling_rate"], return_tensors="pt"
        ... )
        >>> input_features = inputs.input_features
        ...
        >>> with torch.no_grad():
        ...     logits = model(input_features).logits
        ...
        >>> predicted_class_ids = torch.argmax(logits).item()
        >>> predicted_label = model.config.id2label[predicted_class_ids]
        >>> predicted_label
        'Afrikaans'
        ```
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if encoder_outputs is None:
        encoder_outputs = self.encoder(
            input_features,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

    if self.config.use_weighted_layer_sum:
        hidden_states = ops.stack(encoder_outputs, axis=1)
        norm_weights = ops.softmax(self.layer_weights, axis=-1)
        hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(axis=1)
    else:
        hidden_states = encoder_outputs[0]

    hidden_states = self.projector(hidden_states)
    pooled_output = hidden_states.mean(axis=1)

    logits = self.classifier(pooled_output)

    loss = None

    if labels is not None:
        loss = ops.cross_entropy(logits.view(-1, self.config.num_labels), labels.view(-1))

    if not return_dict:
        output = (logits,) + encoder_outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return SequenceClassifierOutput(
        loss=loss,
        logits=logits,
        hidden_states=encoder_outputs.hidden_states,
        attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.freeze_encoder()

Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will not be updated during training. Only the projection layers and classification head will be updated.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def freeze_encoder(self):
    """
    Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
    not be updated during training. Only the projection layers and classification head will be updated.
    """
    self.encoder._freeze_parameters()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.get_input_embeddings()

This method returns the input embeddings from the encoder for audio classification.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class. TYPE: WhisperForAudioClassification

RETURNS	DESCRIPTION
Module	nn.Module: The input embeddings from the encoder for audio classification.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Module:
    """
    This method returns the input embeddings from the encoder for audio classification.

    Args:
        self (WhisperForAudioClassification): The instance of the WhisperForAudioClassification class.

    Returns:
        nn.Module: The input embeddings from the encoder for audio classification.

    Raises:
        None
    """
    return self.encoder.get_input_embeddings()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForAudioClassification.set_input_embeddings(value)

Method to set the input embeddings for the WhisperForAudioClassification class.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForAudioClassification class. Type: WhisperForAudioClassification Purpose: Represents the current instance of the class. Restrictions: None
value	The input embeddings to be set for the encoder. Type: nn.Module Purpose: Represents the input embeddings used for encoding. Restrictions: Should be an instance of nn.Module. TYPE: Module

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value: nn.Module):
    """
    Method to set the input embeddings for the WhisperForAudioClassification class.

    Args:
        self:
            The instance of the WhisperForAudioClassification class.

            - Type: WhisperForAudioClassification
            - Purpose: Represents the current instance of the class.
            - Restrictions: None

        value:
            The input embeddings to be set for the encoder.

            - Type: nn.Module
            - Purpose: Represents the input embeddings used for encoding.
            - Restrictions: Should be an instance of nn.Module.

    Returns:
        None.

    Raises:
        None.
    """
    self.encoder.set_input_embeddings(value)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM

Bases: WhisperPreTrainedModel

WhisperForCausalLM is a class representing a Whisper model for causal language modeling tasks. It inherits from WhisperPreTrainedModel and provides methods for generating text based on input sequences.

METHOD	DESCRIPTION
__init__	Initializes the WhisperForCausalLM model with the given configuration.
get_output_embeddings	Returns the output embeddings of the model.
set_output_embeddings	Sets new output embeddings for the model.
get_input_embeddings	Returns the input embeddings of the model.
set_input_embeddings	Sets new input embeddings for the model.
set_decoder	Sets the decoder for the model.
get_decoder	Returns the decoder of the model.
forward	Constructs the model architecture and processes input data for generation.
prepare_inputs_for_generation	Prepares inputs for text generation based on the provided parameters.
_reorder_cache	Reorders cache items based on a given beam index for generation.

Example

>>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
>>> from datasets import load_dataset
...
>>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
...
>>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> sample = ds[0]["audio"]
>>> input_features = processor(
...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
... ).input_features
...
>>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
...
>>> # Decode token ids to text
>>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
>>> transcription
' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForCausalLM(WhisperPreTrainedModel):

    """
    WhisperForCausalLM is a class representing a Whisper model for causal language modeling tasks.
    It inherits from WhisperPreTrainedModel and provides methods for generating text based on input sequences.

    Methods:
        __init__: Initializes the WhisperForCausalLM model with the given configuration.
        get_output_embeddings: Returns the output embeddings of the model.
        set_output_embeddings: Sets new output embeddings for the model.
        get_input_embeddings: Returns the input embeddings of the model.
        set_input_embeddings: Sets new input embeddings for the model.
        set_decoder: Sets the decoder for the model.
        get_decoder: Returns the decoder of the model.
        forward: Constructs the model architecture and processes input data for generation.
        prepare_inputs_for_generation: Prepares inputs for text generation based on the provided parameters.
        _reorder_cache: Reorders cache items based on a given beam index for generation.

    Example:
        ```python
        >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
        >>> from datasets import load_dataset
        ...
        >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
        ...
        >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> sample = ds[0]["audio"]
        >>> input_features = processor(
        ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
        ... ).input_features
        ...
        >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
        ...
        >>> # Decode token ids to text
        >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
        ```
    """
    _tied_weights_keys = ["proj_out.weight"]
    main_input_name = "input_ids"

    def __init__(self, config):
        """
        Initializes an instance of the WhisperForCausalLM class.

        Args:
            self (WhisperForCausalLM): The instance of the class.
            config: A configuration object containing various settings for the model.

        Returns:
            None

        Raises:
            None
        """
        super().__init__(config)
        config.is_encoder_decoder = False
        self.model = WhisperDecoderWrapper(config)

        self.proj_out = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_output_embeddings(self):
        """
        This method returns the output embeddings for WhisperForCausalLM model.

        Args:
            self: The instance of WhisperForCausalLM class.

        Returns:
            proj_out: This method returns the output embeddings.

        Raises:
            None.
        """
        return self.proj_out

    def set_output_embeddings(self, new_embeddings):
        """
        Set the output embeddings for the WhisperForCausalLM model.

        Args:
            self (WhisperForCausalLM): The instance of WhisperForCausalLM class.
            new_embeddings (Any): The new embeddings to be set as the output embeddings for the model.

        Returns:
            None.

        Raises:
            TypeError: If the new_embeddings parameter is not of the correct type.
            ValueError: If any restrictions or validations fail during the setting of new embeddings.
        """
        self.proj_out = new_embeddings

    def get_input_embeddings(self) -> nn.Module:
        """
        Retrieves the input embeddings from the underlying model.

        Args:
            self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.

        Returns:
            nn.Module: The input embeddings obtained from the underlying model.

        Raises:
            None.

        Description:
            This method returns the input embeddings of the WhisperForCausalLM model.
            The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors.
            The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

        Note:
            - The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature
            extraction.
            - It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

        Example:
            ```python
            >>> model = WhisperForCausalLM()
            >>> embeddings = model.get_input_embeddings()
            ```
        """
        return self.model.get_input_embeddings()

    def set_input_embeddings(self, value):
        """
        Sets the input embeddings for the WhisperForCausalLM model.

        Args:
            self: The object instance.
            value: A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model.
                The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of
                the hidden states in the model. The input embeddings are used to encode the input tokens in the model's
                forward pass. This parameter is required.

        Returns:
            None.

        Raises:
            None.
        """
        self.model.set_input_embeddings(value)

    def set_decoder(self, decoder):
        """
        Method to set the decoder for the WhisperForCausalLM model.

        Args:
            self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.
            decoder: The decoder to be set for the model. It should be compatible with the model's decoder architecture.

        Returns:
            None.

        Raises:
            None.
        """
        self.model.decoder = decoder

    def get_decoder(self):
        """
        Returns the decoder of the WhisperForCausalLM model.

        Args:
            self: The instance of the WhisperForCausalLM class.

        Returns:
            decoder: This method returns the decoder of the WhisperForCausalLM model.

        Raises:
            None.
        """
        return self.model.decoder

    def forward(
        self,
        input_ids: mindspore.Tensor = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
        r"""
        Args:
            input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
                provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
            attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            encoder_outputs  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
                if the model is configured as a decoder.
            head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
                Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.
            past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed
                or when `config.use_cache=True`):
                Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
                shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
                shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
                tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains
                pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
                blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If
                `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
                don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
                `decoder_input_ids` of shape `(batch_size, sequence_length)`.
            inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

        Returns:
            Union[Tuple, CausalLMOutputWithCrossAttentions]

        Example:
            ```python
            >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
            >>> from datasets import load_dataset
            ...
            >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
            ...
            >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
            ...
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            >>> sample = ds[0]["audio"]
            >>> input_features = processor(
            ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
            ... ).input_features
            ...
            >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
            ...
            >>> # decode token ids to text
            >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
            >>> transcription
            ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
            ```
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        # If the user passed a tuple or `BaseModelOutput` for encoder_outputs, we extract only the hidden states
        if isinstance(encoder_outputs, (BaseModelOutput, tuple, list)):
            encoder_outputs = encoder_outputs[0]

        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
        outputs = self.model.decoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_outputs,
            head_mask=head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        logits = self.proj_out(outputs[0])

        loss = None
        if labels is not None:
            loss = ops.cross_entropy(logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[1:]
            return (loss,) + output if loss is not None else output

        return CausalLMOutputWithCrossAttentions(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            cross_attentions=outputs.cross_attentions,
        )

    def prepare_inputs_for_generation(
        self,
        input_ids,
        past_key_values=None,
        use_cache=None,
        encoder_outputs=None,
        attention_mask=None,
        **kwargs,
    ):
        """
        Prepare inputs for generation.

        Args:
            self (object): The instance of the class.
            input_ids (Tensor): The input tensor containing the token ids.
            past_key_values (tuple, optional): The past key values for efficient generation. Defaults to None.
            use_cache (bool, optional): Whether to use caching for the generation process. Defaults to None.
            encoder_outputs (Tensor, optional): The outputs of the encoder. Defaults to None.
            attention_mask (Tensor, optional): The attention mask for the input_ids. Defaults to None.

        Returns:
            dict: A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values,
                input_ids, use_cache, and attention_mask.

        Raises:
            ValueError: If the input_ids and past_key_values are not of compatible shape.
            IndexError: If the input_ids shape is not as expected.
        """
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]

        return {
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "input_ids": input_ids,
            "use_cache": use_cache,
            "attention_mask": attention_mask,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        """
        Reorders the cache of past key values for the beam search in the WhisperForCausalLM class.

        Args:
            past_key_values (tuple): A tuple containing the past key values for each layer.
                Each element in the tuple is expected to be a tensor.
            beam_idx (tensor): The indices of the beams for reordering the past key values.

        Returns:
            None: This method modifies the past_key_values in place.

        Raises:
            None.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.__init__(config)

Initializes an instance of the WhisperForCausalLM class.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: WhisperForCausalLM
config	A configuration object containing various settings for the model.

RETURNS	DESCRIPTION
	None

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config):
    """
    Initializes an instance of the WhisperForCausalLM class.

    Args:
        self (WhisperForCausalLM): The instance of the class.
        config: A configuration object containing various settings for the model.

    Returns:
        None

    Raises:
        None
    """
    super().__init__(config)
    config.is_encoder_decoder = False
    self.model = WhisperDecoderWrapper(config)

    self.proj_out = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.forward(input_ids=None, attention_mask=None, encoder_outputs=None, head_mask=None, cross_attn_head_mask=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
input_ids	Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [AutoTokenizer]. See [PreTrainedTokenizer.encode] and [PreTrainedTokenizer.__call__] for details. What are input IDs? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)` DEFAULT: None
attention_mask	Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]: 1 for tokens that are not masked, 0 for tokens that are masked. What are attention masks? TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
encoder_outputs	Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. TYPE: (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
head_mask	Mask to nullify selected heads of the attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
cross_attn_head_mask	Mask to nullify selected heads of the cross-attention modules. Mask values selected in [0, 1]: 1 indicates the head is not masked, 0 indicates the head is masked. TYPE: `mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional* DEFAULT: None
inputs_embeds	Optionally, instead of passing input_ids you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert input_ids indices into associated vectors than the model's internal embedding lookup matrix. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional* DEFAULT: None
labels	Labels for computing the masked language modeling loss. Indices should either be in [0, ..., config.vocab_size] or -100 (see input_ids docstring). Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None
use_cache	If set to True, past_key_values key value states are returned and can be used to speed up decoding (see past_key_values). 1 for tokens that are not masked, 0 for tokens that are masked. TYPE: `bool`, *optional* DEFAULT: None
output_attentions	Whether or not to return the attentions tensors of all attention layers. See attentions under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
output_hidden_states	Whether or not to return the hidden states of all layers. See hidden_states under returned tensors for more detail. TYPE: `bool`, *optional* DEFAULT: None
return_dict	Whether or not to return a [~utils.ModelOutput] instead of a plain tuple. TYPE: `bool`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple, CausalLMOutputWithCrossAttentions]	Union[Tuple, CausalLMOutputWithCrossAttentions]

Example

>>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
>>> from datasets import load_dataset
...
>>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
...
>>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> sample = ds[0]["audio"]
>>> input_features = processor(
...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
... ).input_features
...
>>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
...
>>> # decode token ids to text
>>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
>>> transcription
' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    input_ids: mindspore.Tensor = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
    r"""
    Args:
        input_ids (`mindspore.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
            provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
        attention_mask (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        encoder_outputs  (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            if the model is configured as a decoder.
        head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        cross_attn_head_mask (`mindspore.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        past_key_values (`tuple(tuple(mindspore.Tensor))`, *optional*, returned when `use_cache=True` is passed
            or when `config.use_cache=True`):
            Tuple of `tuple(mindspore.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
            shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
            tensors are only required when the model is used as a decoder in a Sequence to Sequence model. Contains
            pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If
            `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`mindspore.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
            (see `past_key_values`).

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under
            returned tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
            for more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

    Returns:
        Union[Tuple, CausalLMOutputWithCrossAttentions]

    Example:
        ```python
        >>> from transformers import WhisperForCausalLM, WhisperForConditionalGeneration, WhisperProcessor
        >>> from datasets import load_dataset
        ...
        >>> processor = WhisperProcessor.from_pretrained("openai/whisper-large-v2")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-large-v2")
        ...
        >>> assistant_model = WhisperForCausalLM.from_pretrained("distil-whisper/distil-large-v2")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> sample = ds[0]["audio"]
        >>> input_features = processor(
        ...     sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
        ... ).input_features
        ...
        >>> predicted_ids = model.generate(input_features, assistant_model=assistant_model)
        ...
        >>> # decode token ids to text
        >>> transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'
        ```
    """
    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
    output_hidden_states = (
        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
    )
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    # If the user passed a tuple or `BaseModelOutput` for encoder_outputs, we extract only the hidden states
    if isinstance(encoder_outputs, (BaseModelOutput, tuple, list)):
        encoder_outputs = encoder_outputs[0]

    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
    outputs = self.model.decoder(
        input_ids=input_ids,
        attention_mask=attention_mask,
        encoder_hidden_states=encoder_outputs,
        head_mask=head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )

    logits = self.proj_out(outputs[0])

    loss = None
    if labels is not None:
        loss = ops.cross_entropy(logits.view(-1, self.config.vocab_size), labels.view(-1))

    if not return_dict:
        output = (logits,) + outputs[1:]
        return (loss,) + output if loss is not None else output

    return CausalLMOutputWithCrossAttentions(
        loss=loss,
        logits=logits,
        past_key_values=outputs.past_key_values,
        hidden_states=outputs.hidden_states,
        attentions=outputs.attentions,
        cross_attentions=outputs.cross_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_decoder()

Returns the decoder of the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class.

RETURNS	DESCRIPTION
decoder	This method returns the decoder of the WhisperForCausalLM model.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_decoder(self):
    """
    Returns the decoder of the WhisperForCausalLM model.

    Args:
        self: The instance of the WhisperForCausalLM class.

    Returns:
        decoder: This method returns the decoder of the WhisperForCausalLM model.

    Raises:
        None.
    """
    return self.model.decoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_input_embeddings()

Retrieves the input embeddings from the underlying model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class. TYPE: WhisperForCausalLM

RETURNS	DESCRIPTION
Module	nn.Module: The input embeddings obtained from the underlying model.

Description

This method returns the input embeddings of the WhisperForCausalLM model. The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors. The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

Note

• The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature extraction.
• It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

Example

>>> model = WhisperForCausalLM()
>>> embeddings = model.get_input_embeddings()

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_input_embeddings(self) -> nn.Module:
    """
    Retrieves the input embeddings from the underlying model.

    Args:
        self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.

    Returns:
        nn.Module: The input embeddings obtained from the underlying model.

    Raises:
        None.

    Description:
        This method returns the input embeddings of the WhisperForCausalLM model.
        The input embeddings are responsible for mapping the input tokens to their corresponding embedding vectors.
        The underlying model's 'get_input_embeddings' function is called to retrieve these embeddings.

    Note:
        - The returned input embeddings can be used for various downstream tasks such as fine-tuning or feature
        extraction.
        - It is assumed that the underlying model has a 'get_input_embeddings' method implemented.

    Example:
        ```python
        >>> model = WhisperForCausalLM()
        >>> embeddings = model.get_input_embeddings()
        ```
    """
    return self.model.get_input_embeddings()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.get_output_embeddings()

This method returns the output embeddings for WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of WhisperForCausalLM class.

RETURNS	DESCRIPTION
proj_out	This method returns the output embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_output_embeddings(self):
    """
    This method returns the output embeddings for WhisperForCausalLM model.

    Args:
        self: The instance of WhisperForCausalLM class.

    Returns:
        proj_out: This method returns the output embeddings.

    Raises:
        None.
    """
    return self.proj_out

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.prepare_inputs_for_generation(input_ids, past_key_values=None, use_cache=None, encoder_outputs=None, attention_mask=None, **kwargs)

Prepare inputs for generation.

PARAMETER	DESCRIPTION
self	The instance of the class. TYPE: object
input_ids	The input tensor containing the token ids. TYPE: Tensor
past_key_values	The past key values for efficient generation. Defaults to None. TYPE: tuple DEFAULT: None
use_cache	Whether to use caching for the generation process. Defaults to None. TYPE: bool DEFAULT: None
encoder_outputs	The outputs of the encoder. Defaults to None. TYPE: Tensor DEFAULT: None
attention_mask	The attention mask for the input_ids. Defaults to None. TYPE: Tensor DEFAULT: None

RETURNS	DESCRIPTION
dict	A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values, input_ids, use_cache, and attention_mask.

RAISES	DESCRIPTION
ValueError	If the input_ids and past_key_values are not of compatible shape.
IndexError	If the input_ids shape is not as expected.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def prepare_inputs_for_generation(
    self,
    input_ids,
    past_key_values=None,
    use_cache=None,
    encoder_outputs=None,
    attention_mask=None,
    **kwargs,
):
    """
    Prepare inputs for generation.

    Args:
        self (object): The instance of the class.
        input_ids (Tensor): The input tensor containing the token ids.
        past_key_values (tuple, optional): The past key values for efficient generation. Defaults to None.
        use_cache (bool, optional): Whether to use caching for the generation process. Defaults to None.
        encoder_outputs (Tensor, optional): The outputs of the encoder. Defaults to None.
        attention_mask (Tensor, optional): The attention mask for the input_ids. Defaults to None.

    Returns:
        dict: A dictionary containing the prepared inputs for generation including encoder_outputs, past_key_values,
            input_ids, use_cache, and attention_mask.

    Raises:
        ValueError: If the input_ids and past_key_values are not of compatible shape.
        IndexError: If the input_ids shape is not as expected.
    """
    if past_key_values is not None:
        past_length = past_key_values[0][0].shape[2]

        # Some generation methods already pass only the last input ID
        if input_ids.shape[1] > past_length:
            remove_prefix_length = past_length
        else:
            # Default to old behavior: keep only final ID
            remove_prefix_length = input_ids.shape[1] - 1

        input_ids = input_ids[:, remove_prefix_length:]

    return {
        "encoder_outputs": encoder_outputs,
        "past_key_values": past_key_values,
        "input_ids": input_ids,
        "use_cache": use_cache,
        "attention_mask": attention_mask,
    }

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_decoder(decoder)

Method to set the decoder for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of the WhisperForCausalLM class. TYPE: WhisperForCausalLM
decoder	The decoder to be set for the model. It should be compatible with the model's decoder architecture.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_decoder(self, decoder):
    """
    Method to set the decoder for the WhisperForCausalLM model.

    Args:
        self (WhisperForCausalLM): The instance of the WhisperForCausalLM class.
        decoder: The decoder to be set for the model. It should be compatible with the model's decoder architecture.

    Returns:
        None.

    Raises:
        None.
    """
    self.model.decoder = decoder

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_input_embeddings(value)

Sets the input embeddings for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The object instance.
value	A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model. The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of the hidden states in the model. The input embeddings are used to encode the input tokens in the model's forward pass. This parameter is required.

RETURNS	DESCRIPTION
	None.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_input_embeddings(self, value):
    """
    Sets the input embeddings for the WhisperForCausalLM model.

    Args:
        self: The object instance.
        value: A tensor of shape (vocab_size, hidden_size) representing the new input embeddings for the model.
            The vocab_size is the size of the vocabulary used by the model, and the hidden_size is the size of
            the hidden states in the model. The input embeddings are used to encode the input tokens in the model's
            forward pass. This parameter is required.

    Returns:
        None.

    Raises:
        None.
    """
    self.model.set_input_embeddings(value)

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForCausalLM.set_output_embeddings(new_embeddings)

Set the output embeddings for the WhisperForCausalLM model.

PARAMETER	DESCRIPTION
self	The instance of WhisperForCausalLM class. TYPE: WhisperForCausalLM
new_embeddings	The new embeddings to be set as the output embeddings for the model. TYPE: Any

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
TypeError	If the new_embeddings parameter is not of the correct type.
ValueError	If any restrictions or validations fail during the setting of new embeddings.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def set_output_embeddings(self, new_embeddings):
    """
    Set the output embeddings for the WhisperForCausalLM model.

    Args:
        self (WhisperForCausalLM): The instance of WhisperForCausalLM class.
        new_embeddings (Any): The new embeddings to be set as the output embeddings for the model.

    Returns:
        None.

    Raises:
        TypeError: If the new_embeddings parameter is not of the correct type.
        ValueError: If any restrictions or validations fail during the setting of new embeddings.
    """
    self.proj_out = new_embeddings

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration

Bases: WhisperPreTrainedModel

The WhisperForConditionalGeneration class is a model class for conditional text generation, inheriting from WhisperPreTrainedModel. It provides methods for initializing the model, generating sequences of token ids, preparing inputs for generation, and extracting token-level timestamps for predicted tokens.

The class contains methods such as forward, generate, prepare_inputs_for_generation, and _reorder_cache for handling conditional generation tasks. It also includes methods for freezing the encoder, getting the encoder and decoder, and managing the input and output embeddings.

The class's main methods include:

• forward: Prepares inputs and generates sequences of token ids for conditional text generation, allowing for the configuration of various generation parameters.
• generate: Generates sequences of token ids for models with a language modeling head, allowing for custom logits processors, stopping criteria, and other advanced generation parameters.
• prepare_inputs_for_generation: Prepares input data for generation, including decoder input ids, past key values, cache usage, encoder outputs, and attention masks.
• _reorder_cache: Reorders the past key values based on beam indices during generation.
• _extract_token_timestamps: Calculates token-level timestamps using encoder-decoder cross-attentions and dynamic time-warping (DTW) to map each output token to a position in the input audio.

This class provides a comprehensive set of tools for conditional text generation tasks, including multilingual and multitask generation support, as well as token-level timestamps extraction for predicted tokens.

For more details on how to use the class and its methods, including code examples, refer to the official documentation and the following guide.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

class WhisperForConditionalGeneration(WhisperPreTrainedModel):

    """
    The `WhisperForConditionalGeneration` class is a model class for conditional text generation, inheriting from 
    `WhisperPreTrainedModel`. It provides methods for initializing the model, generating sequences of token ids, 
    preparing inputs for generation, and extracting token-level timestamps for predicted tokens.

    The class contains methods such as `forward`, `generate`, `prepare_inputs_for_generation`, and `_reorder_cache` 
    for handling conditional generation tasks. It also includes methods for freezing the encoder, getting the encoder 
    and decoder, and managing the input and output embeddings.

    The class's main methods include:

    - `forward`: Prepares inputs and generates sequences of token ids for conditional text generation, allowing for 
    the configuration of various generation parameters.
    - `generate`: Generates sequences of token ids for models with a language modeling head, allowing for custom logits 
    processors, stopping criteria, and other advanced generation parameters.
    - `prepare_inputs_for_generation`: Prepares input data for generation, including decoder input ids, past key values, 
    cache usage, encoder outputs, and attention masks.
    - `_reorder_cache`: Reorders the past key values based on beam indices during generation.
    - `_extract_token_timestamps`: Calculates token-level timestamps using encoder-decoder cross-attentions and dynamic 
    time-warping (DTW) to map each output token to a position in the input audio.

    This class provides a comprehensive set of tools for conditional text generation tasks, including multilingual 
    and multitask generation support, as well as token-level timestamps extraction for predicted tokens.

    For more details on how to use the class and its methods, including code examples, refer to the official 
    documentation and the [following guide](./generation_strategies).
    """
    base_model_prefix = "model"
    _tied_weights_keys = ["proj_out.weight"]

    def __init__(self, config: WhisperConfig):
        """
        Initializes an instance of the WhisperForConditionalGeneration class.

        Args:
            self (WhisperForConditionalGeneration): The instance of the class itself.
            config (WhisperConfig): An instance of WhisperConfig containing configuration parameters for the model.

        Returns:
            None.

        Raises:
            AssertionError: If the config parameter is not of type WhisperConfig.
            ValueError: If an unexpected error occurs during initialization.
        """
        super().__init__(config)
        self.model = WhisperModel(config)
        self.proj_out = nn.Linear(config.d_model, config.vocab_size, bias=False)

        # Initialize weights and apply final processing
        self.post_init()

    def get_encoder(self):
        """
        Retrieves the encoder from the model instance.

        Args:
            self (WhisperForConditionalGeneration): The object instance.

        Returns:
            None.

        Raises:
            None.

        """
        return self.model.get_encoder()

    def get_decoder(self):
        """
        This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves 
        the decoder from the model.

        Args:
            self:
                Instance of the 'WhisperForConditionalGeneration' class.

                - Type: object
                - Purpose: Represents the current instance of the class.
                - Restrictions: This parameter is required for accessing the decoder.

        Returns:
            None:

                - Type: None
                - Purpose: The method returns None as it retrieves the decoder from the model.

        Raises:
            None.
        """
        return self.model.get_decoder()

    def get_output_embeddings(self):
        """
        Method to retrieve the output embeddings from the WhisperForConditionalGeneration class.

        Args:
            self:
                An instance of the WhisperForConditionalGeneration class.

                - Type: WhisperForConditionalGeneration
                - Purpose: Represents the current object of the class.
                - Restrictions: Must be an instance of WhisperForConditionalGeneration class.

        Returns:
            None.

        Raises:
            None.
        """
        return self.proj_out

    def set_output_embeddings(self, new_embeddings):
        """
        This method sets the output embeddings for the WhisperForConditionalGeneration class.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
            new_embeddings (any): The new embeddings to be set as the output embeddings for the 
                WhisperForConditionalGeneration class. It can be of any type.

        Returns:
            None.

        Raises:
            None.
        """
        self.proj_out = new_embeddings

    def get_input_embeddings(self) -> nn.Module:
        """
        Returns the input embeddings for the WhisperForConditionalGeneration model.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.

        Returns:
            nn.Module: The input embeddings for the model.

        Raises:
            None.
        """
        return self.model.get_input_embeddings()

    def freeze_encoder(self):
        """
        Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
        not be updated during training.
        """
        self.model.encoder._freeze_parameters()

    def forward(
        self,
        input_features: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        head_mask: Optional[mindspore.Tensor] = None,
        decoder_head_mask: Optional[mindspore.Tensor] = None,
        cross_attn_head_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
        r"""
        Args:
            labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]`
                or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is
                only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:
            Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

        Example:
            ```python
            >>> from transformers import AutoProcessor, WhisperForConditionalGeneration
            >>> from datasets import load_dataset
            ...
            >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
            ...
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            ...
            >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
            >>> input_features = inputs.input_features
            ...
            >>> generated_ids = model.generate(inputs=input_features)
            ...
            >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
            >>> transcription
            ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
            ```
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_features,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        lm_logits = self.proj_out(outputs[0])

        loss = None
        if labels is not None:
            loss = ops.cross(lm_logits.view(-1, self.config.vocab_size), labels.reshape(-1))

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((loss,) + output) if loss is not None else output

        return Seq2SeqLMOutput(
            loss=loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,
            decoder_hidden_states=outputs.decoder_hidden_states,
            decoder_attentions=outputs.decoder_attentions,
            cross_attentions=outputs.cross_attentions,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
        )

    def generate(
        self,
        inputs: Optional[mindspore.Tensor] = None,
        generation_config=None,
        logits_processor=None,
        stopping_criteria=None,
        prefix_allowed_tokens_fn=None,
        synced_gpus=False,
        return_timestamps=None,
        task=None,
        language=None,
        is_multilingual=None,
        prompt_ids: Optional[mindspore.Tensor] = None,
        return_token_timestamps=None,
        **kwargs,
    ):
        """

        Generates sequences of token ids for models with a language modeling head.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

        For an overview of generation strategies and code examples, check out the [following
        guide](./generation_strategies).

        </Tip>

        Parameters:
            inputs (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
                The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
                method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
                should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
                `input_ids`, `input_values`, `input_features`, or `pixel_values`.
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            logits_processor (`LogitsProcessorList`, *optional*):
                Custom logits processors that complement the default logits processors built from arguments and
                generation config. If a logit processor is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            stopping_criteria (`StoppingCriteriaList`, *optional*):
                Custom stopping criteria that complement the default stopping criteria built from arguments and a
                generation config. If a stopping criteria is passed that is already created with the arguments or a
                generation config an error is thrown. This feature is intended for advanced users.
            prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
                If provided, this function constraints the beam search to allowed tokens only at each step. If not
                provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
                `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
                on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
                for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
                Retrieval](https://arxiv.org/abs/2010.00904).
            synced_gpus (`bool`, *optional*, defaults to `False`):
                Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
            return_timestamps (`bool`, *optional*):
                Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`.
            task (`str`, *optional*):
                Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids`
                will be updated accordingly.
            language (`str`, *optional*):
                Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can
                find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary.
            is_multilingual (`bool`, *optional*):
                Whether or not the model is multilingual.
            prompt_ids (`mindspore.Tensor`, *optional*):
                Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is
                provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for
                transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words
                correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value.
            return_token_timestamps (`bool`, *optional*):
                Whether to return token-level timestamps with the text. This can be used with or without the
                `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into
                words.
            kwargs (`Dict[str, Any]`, *optional*):
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
                specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.

        Returns:
            [`~utils.ModelOutput`] or `mindspore.Tensor`:
                A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` or when
                `config.return_dict_in_generate=True`) or a `mindspore.Tensor`.
                If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
                [`~utils.ModelOutput`] types are:

                - [`~generation.GreedySearchDecoderOnlyOutput`],
                - [`~generation.SampleDecoderOnlyOutput`],
                - [`~generation.BeamSearchDecoderOnlyOutput`],
                - [`~generation.BeamSampleDecoderOnlyOutput`]

                If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
                [`~utils.ModelOutput`] types are:

                - [`~generation.GreedySearchEncoderDecoderOutput`],
                - [`~generation.SampleEncoderDecoderOutput`],
                - [`~generation.BeamSearchEncoderDecoderOutput`],
                - [`~generation.BeamSampleEncoderDecoderOutput`]
        """
        if generation_config is None:
            generation_config = self.generation_config

        if return_timestamps is not None:
            if not hasattr(generation_config, "no_timestamps_token_id"):
                raise ValueError(
                    "You are trying to return timestamps, but the generation config is not properly set. "
                    "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. "
                    "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363"
                )

            generation_config.return_timestamps = return_timestamps
        else:
            generation_config.return_timestamps = False

        if is_multilingual is not None:
            if not hasattr(generation_config, "is_multilingual"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `is_multilingual` argument "
                    "to `generate`. Please update the generation config as per the instructions "
                    "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            generation_config.is_multilingual = is_multilingual

        if hasattr(generation_config, "is_multilingual") and not generation_config.is_multilingual:
            if task is not None or language is not None:
                raise ValueError(
                    "Cannot specify `task` or `language` for an English-only model. If the model is intended to be "
                    "multilingual, pass `is_multilingual=True` to generate, or update the generation config."
                )

        if language is not None:
            if not hasattr(generation_config, "lang_to_id"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `language` argument "
                    "to `generate`. Either set the language using the `forced_decoder_ids` in the model config, "
                    "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            language = language.lower()
            generation_config.language = language
        if task is not None:
            if not hasattr(generation_config, "task_to_id"):
                raise ValueError(
                    "The generation config is outdated and is thus not compatible with the `task` argument "
                    "to `generate`. Either set the task using the `forced_decoder_ids` in the model config, "
                    "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
                )
            generation_config.task = task

        forced_decoder_ids = None

        # Legacy code for backward compatibility
        if hasattr(self.config, "forced_decoder_ids") and self.config.forced_decoder_ids is not None:
            forced_decoder_ids = self.config.forced_decoder_ids
        elif (
            hasattr(self.generation_config, "forced_decoder_ids")
            and self.generation_config.forced_decoder_ids is not None
        ):
            forced_decoder_ids = self.generation_config.forced_decoder_ids
        else:
            forced_decoder_ids = kwargs.get("forced_decoder_ids", None)

        if task is not None or language is not None or (forced_decoder_ids is None and prompt_ids is not None):
            forced_decoder_ids = []
            if hasattr(generation_config, "language"):
                if generation_config.language in generation_config.lang_to_id.keys():
                    language_token = generation_config.language
                elif generation_config.language in TO_LANGUAGE_CODE.keys():
                    language_token = f"<|{TO_LANGUAGE_CODE[generation_config.language]}|>"
                elif generation_config.language in TO_LANGUAGE_CODE.values():
                    language_token = f"<|{generation_config.language}|>"
                else:
                    is_language_code = len(generation_config.language) == 2
                    raise ValueError(
                        f"Unsupported language: {generation_config.language}. Language should be one of:"
                        f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                    )
                forced_decoder_ids.append((1, generation_config.lang_to_id[language_token]))
            else:
                forced_decoder_ids.append((1, None))  # automatically detect the language

            if hasattr(generation_config, "task"):
                if generation_config.task in TASK_IDS:
                    forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task]))
                else:
                    raise ValueError(
                        f"The `{generation_config.task}`task is not supported. The task should be one of `{TASK_IDS}`"
                    )
            elif hasattr(generation_config, "task_to_id"):
                forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"]))  # defaults to transcribe
            if hasattr(generation_config, "no_timestamps_token_id") and not generation_config.return_timestamps:
                idx = forced_decoder_ids[-1][0] + 1 if forced_decoder_ids else 1
                forced_decoder_ids.append((idx, generation_config.no_timestamps_token_id))

        if forced_decoder_ids is not None:
            generation_config.forced_decoder_ids = forced_decoder_ids

        if prompt_ids is not None:
            if kwargs.get("decoder_start_token_id") is not None:
                raise ValueError(
                    "When specifying `prompt_ids`, you cannot also specify `decoder_start_token_id` as it gets overwritten."
                )
            prompt_ids = prompt_ids.tolist()
            decoder_start_token_id, *text_prompt_ids = prompt_ids
            # Slicing the text prompt ids in a manner consistent with the OpenAI implementation
            # to accomodate context space for the prefix (see https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/decoding.py#L599)
            text_prompt_ids = text_prompt_ids[-self.config.max_target_positions // 2 - 1 :]
            # Set the decoder_start_token_id to <|startofprev|>
            kwargs.update({"decoder_start_token_id": decoder_start_token_id})

            # If the user passes `max_new_tokens`, increase its number to account for the prompt
            if kwargs.get("max_new_tokens", None) is not None:
                kwargs["max_new_tokens"] += len(text_prompt_ids)
                if kwargs["max_new_tokens"] >= self.config.max_target_positions:
                    raise ValueError(
                        f"The length of the sliced `prompt_ids` is {len(text_prompt_ids)}, and the `max_new_tokens` "
                        f"{kwargs['max_new_tokens'] - len(text_prompt_ids)}. Thus, the combined length of the sliced "
                        f"`prompt_ids` and `max_new_tokens` is: {kwargs['max_new_tokens']}. This exceeds the "
                        f"`max_target_positions` of the Whisper model: {self.config.max_target_positions}. "
                        "You should either reduce the length of your prompt, or reduce the value of `max_new_tokens`, "
                        f"so that their combined length is less that {self.config.max_target_positions}."
                    )

            # Reformat the forced_decoder_ids to incorporate the prompt
            non_prompt_forced_decoder_ids = (
                kwargs.pop("forced_decoder_ids", None) or generation_config.forced_decoder_ids
            )
            forced_decoder_ids = [
                *text_prompt_ids,
                generation_config.decoder_start_token_id,
                *[token for _rank, token in non_prompt_forced_decoder_ids],
            ]
            forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_decoder_ids)]
            generation_config.forced_decoder_ids = forced_decoder_ids

        if generation_config.return_timestamps:
            logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)]

        if return_token_timestamps:
            kwargs["output_attentions"] = True
            kwargs["return_dict_in_generate"] = True

            if getattr(generation_config, "task", None) == "translate":
                logger.warning("Token-level timestamps may not be reliable for task 'translate'.")
            if not hasattr(generation_config, "alignment_heads"):
                raise ValueError(
                    "Model generation config has no `alignment_heads`, token-level timestamps not available. "
                    "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config."
                )

            if kwargs.get("num_frames") is not None:
                generation_config.num_frames = kwargs.pop("num_frames")

        outputs = super().generate(
            inputs,
            generation_config,
            logits_processor,
            stopping_criteria,
            prefix_allowed_tokens_fn,
            synced_gpus,
            **kwargs,
        )

        if return_token_timestamps and hasattr(generation_config, "alignment_heads"):
            num_frames = getattr(generation_config, "num_frames", None)
            outputs["token_timestamps"] = self._extract_token_timestamps(
                outputs, generation_config.alignment_heads, num_frames=num_frames
            )

        return outputs

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        use_cache=None,
        encoder_outputs=None,
        attention_mask=None,
        **kwargs,
    ):
        """
        Prepare inputs for generation.

        Args:
            self (WhisperForConditionalGeneration): The instance of the WhisperForConditionalGeneration class.
            decoder_input_ids (torch.Tensor): The input tensor for the decoder.
                Shape: (batch_size, sequence_length).
            past_key_values (tuple, optional): The past key values for caching computations in auto-regressive decoding.
                Default: None.
            use_cache (bool, optional): Whether to use caching for fast decoding.
                Default: None.
            encoder_outputs (torch.Tensor, optional): The output of the encoder.
                Shape: (batch_size, sequence_length, hidden_size).
                Default: None.
            attention_mask (torch.Tensor, optional): The attention mask for the decoder input.
                Shape: (batch_size, sequence_length).
                Default: None.
            **kwargs: Additional keyword arguments.

        Returns:
            dict: A dictionary containing the prepared inputs for generation.
                It includes the following keys:

                - 'encoder_outputs' (torch.Tensor): The output of the encoder.
                - 'past_key_values' (tuple): The past key values for caching computations in auto-regressive decoding.
                - 'decoder_input_ids' (torch.Tensor): The input tensor for the decoder.
                - 'use_cache' (bool): Whether to use caching for fast decoding.
                - 'decoder_attention_mask' (None): The attention mask for the decoder input.

        Raises:
            None.
        """
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            # Some generation methods already pass only the last input ID
            if decoder_input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                # Default to old behavior: keep only final ID
                remove_prefix_length = decoder_input_ids.shape[1] - 1

            decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

        return {
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "use_cache": use_cache,
            "decoder_attention_mask": None,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        """
        Reorders the cache according to the provided beam index.

        Args:
            past_key_values (tuple): A tuple containing the past key-value states for each layer.
                Each element in the tuple is a tensor representing the past state of a layer.
            beam_idx (Tensor): A tensor containing the indices of the beams to be reordered.

        Returns:
            tuple: A tuple containing the reordered past key-value states for each layer. Each element in the tuple
                is a tensor representing the reordered past state of a layer.

        Raises:
            None.

        This static method takes the past key-value states and a beam index tensor, and reorders the past key-value
            states according to the beam index. It returns the reordered past key-value states as a
            tuple, where each element in the tuple represents the reordered past state of a layer.

        Note:
            The returned reordered_past tuple has the same length as the number of layers in the model, and each
                element in the tuple has the same shape as the corresponding element in the past_key_values tuple.
        """
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),
            )
        return reordered_past

    def _extract_token_timestamps(self, generate_outputs, alignment_heads, time_precision=0.02, num_frames=None):
        """
        Calculates token-level timestamps using the encoder-decoder cross-attentions and dynamic time-warping (DTW) to
        map each output token to a position in the input audio. If `num_frames` is specified, the encoder-decoder
        cross-attentions will be cropped before applying DTW.

        Returns:
            tensor containing the timestamps in seconds for each predicted token
        """
        # Create a list with `decoder_layers` elements, each a tensor of shape
        # (batch size, attention_heads, output length, input length).
        cross_attentions = []
        for i in range(self.config.decoder_layers):
            cross_attentions.append(ops.cat([x[i] for x in generate_outputs.cross_attentions], axis=2))

        # Select specific cross-attention layers and heads. This is a tensor
        # of shape (batch size, num selected, output length, input length).
        weights = ops.stack([cross_attentions[l][:, h] for l, h in alignment_heads])
        weights = weights.permute([1, 0, 2, 3])
        if num_frames is not None:
            weights = weights[..., : num_frames // 2]

        # Normalize and smoothen the weights.
        std, mean = ops.std_mean(weights, axis=-2, keepdims=True)
        weights = (weights - mean) / std
        weights = _median_filter(weights, self.config.median_filter_width)

        # Average the different cross-attention heads.
        matrix = weights.mean(axis=1)

        timestamps = ops.zeros_like(generate_outputs.sequences, dtype=mindspore.float32)

        # Perform dynamic time warping on each element of the batch.
        for batch_idx in range(timestamps.shape[0]):
            text_indices, time_indices = _dynamic_time_warping(-matrix[batch_idx].asnumpy())
            jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)
            jump_times = time_indices[jumps] * time_precision
            timestamps[batch_idx, 1:] = mindspore.tensor(jump_times)

        return timestamps

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.__init__(config)

Initializes an instance of the WhisperForConditionalGeneration class.

PARAMETER	DESCRIPTION
self	The instance of the class itself. TYPE: WhisperForConditionalGeneration
config	An instance of WhisperConfig containing configuration parameters for the model. TYPE: WhisperConfig

RETURNS	DESCRIPTION
	None.

RAISES	DESCRIPTION
AssertionError	If the config parameter is not of type WhisperConfig.
ValueError	If an unexpected error occurs during initialization.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def __init__(self, config: WhisperConfig):
    """
    Initializes an instance of the WhisperForConditionalGeneration class.

    Args:
        self (WhisperForConditionalGeneration): The instance of the class itself.
        config (WhisperConfig): An instance of WhisperConfig containing configuration parameters for the model.

    Returns:
        None.

    Raises:
        AssertionError: If the config parameter is not of type WhisperConfig.
        ValueError: If an unexpected error occurs during initialization.
    """
    super().__init__(config)
    self.model = WhisperModel(config)
    self.proj_out = nn.Linear(config.d_model, config.vocab_size, bias=False)

    # Initialize weights and apply final processing
    self.post_init()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.forward(input_features=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, cross_attn_head_mask=None, encoder_outputs=None, past_key_values=None, decoder_inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, return_dict=None)

PARAMETER	DESCRIPTION
labels	Labels for computing the language modeling loss. Indices should either be in [0, ..., config.vocab_size] or -100 (see input_ids docstring). Tokens with indices set to -100 are ignored (masked), the loss is only computed for the tokens with labels in [0, ..., config.vocab_size]. TYPE: `mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional* DEFAULT: None

RETURNS	DESCRIPTION
Union[Tuple[Tensor], Seq2SeqLMOutput]	Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

Example

>>> from transformers import AutoProcessor, WhisperForConditionalGeneration
>>> from datasets import load_dataset
...
>>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
>>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
...
>>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_features = inputs.input_features
...
>>> generated_ids = model.generate(inputs=input_features)
...
>>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
>>> transcription
' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def forward(
    self,
    input_features: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    head_mask: Optional[mindspore.Tensor] = None,
    decoder_head_mask: Optional[mindspore.Tensor] = None,
    cross_attn_head_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    decoder_inputs_embeds: Optional[Tuple[mindspore.Tensor]] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    return_dict: Optional[bool] = None,
) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
    r"""
    Args:
        labels (`mindspore.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]`
            or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is
            only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

    Returns:
        Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

    Example:
        ```python
        >>> from transformers import AutoProcessor, WhisperForConditionalGeneration
        >>> from datasets import load_dataset
        ...
        >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")
        >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        ...
        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_features = inputs.input_features
        ...
        >>> generated_ids = model.generate(inputs=input_features)
        ...
        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if labels is not None:
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            decoder_input_ids = shift_tokens_right(
                labels, self.config.pad_token_id, self.config.decoder_start_token_id
            )

    outputs = self.model(
        input_features,
        attention_mask=attention_mask,
        decoder_input_ids=decoder_input_ids,
        encoder_outputs=encoder_outputs,
        decoder_attention_mask=decoder_attention_mask,
        head_mask=head_mask,
        decoder_head_mask=decoder_head_mask,
        cross_attn_head_mask=cross_attn_head_mask,
        past_key_values=past_key_values,
        decoder_inputs_embeds=decoder_inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
    )
    lm_logits = self.proj_out(outputs[0])

    loss = None
    if labels is not None:
        loss = ops.cross(lm_logits.view(-1, self.config.vocab_size), labels.reshape(-1))

    if not return_dict:
        output = (lm_logits,) + outputs[1:]
        return ((loss,) + output) if loss is not None else output

    return Seq2SeqLMOutput(
        loss=loss,
        logits=lm_logits,
        past_key_values=outputs.past_key_values,
        decoder_hidden_states=outputs.decoder_hidden_states,
        decoder_attentions=outputs.decoder_attentions,
        cross_attentions=outputs.cross_attentions,
        encoder_last_hidden_state=outputs.encoder_last_hidden_state,
        encoder_hidden_states=outputs.encoder_hidden_states,
        encoder_attentions=outputs.encoder_attentions,
    )

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.freeze_encoder()

Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will not be updated during training.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def freeze_encoder(self):
    """
    Calling this function will disable the gradient computation for the Whisper encoder so that its parameters will
    not be updated during training.
    """
    self.model.encoder._freeze_parameters()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate(inputs=None, generation_config=None, logits_processor=None, stopping_criteria=None, prefix_allowed_tokens_fn=None, synced_gpus=False, return_timestamps=None, task=None, language=None, is_multilingual=None, prompt_ids=None, return_token_timestamps=None, **kwargs)

Generates sequences of token ids for models with a language modeling head.

Most generation-controlling parameters are set in generation_config which, if not passed, will be set to the model's default generation configuration. You can override any generation_config by passing the corresponding parameters to generate(), e.g. .generate(inputs, num_beams=4, do_sample=True).

For an overview of generation strategies and code examples, check out the following guide.

PARAMETER	DESCRIPTION
inputs	The sequence used as a prompt for the generation or as model inputs to the encoder. If None the method initializes it with bos_token_id and a batch size of 1. For decoder-only models inputs should of in the format of input_ids. For encoder-decoder models inputs can represent any of input_ids, input_values, input_features, or pixel_values. TYPE: `mindspore.Tensor` of varying shape depending on the modality, *optional* DEFAULT: None
generation_config	The generation configuration to be used as base parametrization for the generation call. **kwargs passed to generate matching the attributes of generation_config will override them. If generation_config is not provided, the default will be used, which had the following loading priority: 1) from the generation_config.json model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [~generation.GenerationConfig]'s default values, whose documentation should be checked to parameterize generation. TYPE: `~generation.GenerationConfig`, *optional* DEFAULT: None
logits_processor	Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `LogitsProcessorList`, *optional* DEFAULT: None
stopping_criteria	Custom stopping criteria that complement the default stopping criteria built from arguments and a generation config. If a stopping criteria is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. TYPE: `StoppingCriteriaList`, *optional* DEFAULT: None
prefix_allowed_tokens_fn	If provided, this function constraints the beam search to allowed tokens only at each step. If not provided no constraint is applied. This function takes 2 arguments: the batch ID batch_id and input_ids. It has to return a list with the allowed tokens for the next generation step conditioned on the batch ID batch_id and the previously generated tokens inputs_ids. This argument is useful for constrained generation conditioned on the prefix, as described in Autoregressive Entity Retrieval. TYPE: `Callable[[int, mindspore.Tensor], List[int]]`, *optional* DEFAULT: None
synced_gpus	Whether to continue running the while loop until max_length (needed for ZeRO stage 3) TYPE: `bool`, *optional*, defaults to `False` DEFAULT: False
return_timestamps	Whether to return the timestamps with the text. This enables the WhisperTimestampsLogitsProcessor. TYPE: `bool`, *optional* DEFAULT: None
task	Task to use for generation, either "translate" or "transcribe". The model.config.forced_decoder_ids will be updated accordingly. TYPE: `str`, *optional* DEFAULT: None
language	Language token to use for generation, can be either in the form of <|en|>, en or english. You can find all the possible language tokens in the model.generation_config.lang_to_id dictionary. TYPE: `str`, *optional* DEFAULT: None
is_multilingual	Whether or not the model is multilingual. TYPE: `bool`, *optional* DEFAULT: None
prompt_ids	Rank-1 tensor of token IDs created by passing text to [~WhisperProcessor.get_prompt_ids] that is provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words correctly. It cannot be used in conjunction with decoder_start_token_id as it overwrites this value. TYPE: `mindspore.Tensor`, *optional* DEFAULT: None
return_token_timestamps	Whether to return token-level timestamps with the text. This can be used with or without the return_timestamps option. To get word-level timestamps, use the tokenizer to group the tokens into words. TYPE: `bool`, *optional* DEFAULT: None
kwargs	Ad hoc parametrization of generate_config and/or additional model-specific kwargs that will be forwarded to the forward function of the model. If the model is an encoder-decoder model, encoder specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with decoder_. TYPE: `Dict[str, Any]`, *optional* DEFAULT: {}

RETURNS

DESCRIPTION

[~utils.ModelOutput] or mindspore.Tensor: A [~utils.ModelOutput] (if return_dict_in_generate=True or when config.return_dict_in_generate=True) or a mindspore.Tensor. If the model is not an encoder-decoder model (model.config.is_encoder_decoder=False), the possible [~utils.ModelOutput] types are: [~generation.GreedySearchDecoderOnlyOutput], [~generation.SampleDecoderOnlyOutput], [~generation.BeamSearchDecoderOnlyOutput], [~generation.BeamSampleDecoderOnlyOutput] If the model is an encoder-decoder model (model.config.is_encoder_decoder=True), the possible [~utils.ModelOutput] types are: [~generation.GreedySearchEncoderDecoderOutput], [~generation.SampleEncoderDecoderOutput], [~generation.BeamSearchEncoderDecoderOutput], [~generation.BeamSampleEncoderDecoderOutput]

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def generate(
    self,
    inputs: Optional[mindspore.Tensor] = None,
    generation_config=None,
    logits_processor=None,
    stopping_criteria=None,
    prefix_allowed_tokens_fn=None,
    synced_gpus=False,
    return_timestamps=None,
    task=None,
    language=None,
    is_multilingual=None,
    prompt_ids: Optional[mindspore.Tensor] = None,
    return_token_timestamps=None,
    **kwargs,
):
    """

    Generates sequences of token ids for models with a language modeling head.

    <Tip warning={true}>

    Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
    model's default generation configuration. You can override any `generation_config` by passing the corresponding
    parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`.

    For an overview of generation strategies and code examples, check out the [following
    guide](./generation_strategies).

    </Tip>

    Parameters:
        inputs (`mindspore.Tensor` of varying shape depending on the modality, *optional*):
            The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
            method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
            should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
            `input_ids`, `input_values`, `input_features`, or `pixel_values`.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call. `**kwargs`
            passed to generate matching the attributes of `generation_config` will override them. If
            `generation_config` is not provided, the default will be used, which had the following loading
            priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
            configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
            default values, whose documentation should be checked to parameterize generation.
        logits_processor (`LogitsProcessorList`, *optional*):
            Custom logits processors that complement the default logits processors built from arguments and
            generation config. If a logit processor is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        stopping_criteria (`StoppingCriteriaList`, *optional*):
            Custom stopping criteria that complement the default stopping criteria built from arguments and a
            generation config. If a stopping criteria is passed that is already created with the arguments or a
            generation config an error is thrown. This feature is intended for advanced users.
        prefix_allowed_tokens_fn (`Callable[[int, mindspore.Tensor], List[int]]`, *optional*):
            If provided, this function constraints the beam search to allowed tokens only at each step. If not
            provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
            `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
            on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
            for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
            Retrieval](https://arxiv.org/abs/2010.00904).
        synced_gpus (`bool`, *optional*, defaults to `False`):
            Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
        return_timestamps (`bool`, *optional*):
            Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`.
        task (`str`, *optional*):
            Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids`
            will be updated accordingly.
        language (`str`, *optional*):
            Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can
            find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary.
        is_multilingual (`bool`, *optional*):
            Whether or not the model is multilingual.
        prompt_ids (`mindspore.Tensor`, *optional*):
            Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is
            provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for
            transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words
            correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value.
        return_token_timestamps (`bool`, *optional*):
            Whether to return token-level timestamps with the text. This can be used with or without the
            `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into
            words.
        kwargs (`Dict[str, Any]`, *optional*):
            Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
            forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
            specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.

    Returns:
        [`~utils.ModelOutput`] or `mindspore.Tensor`:
            A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` or when
            `config.return_dict_in_generate=True`) or a `mindspore.Tensor`.
            If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
            [`~utils.ModelOutput`] types are:

            - [`~generation.GreedySearchDecoderOnlyOutput`],
            - [`~generation.SampleDecoderOnlyOutput`],
            - [`~generation.BeamSearchDecoderOnlyOutput`],
            - [`~generation.BeamSampleDecoderOnlyOutput`]

            If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
            [`~utils.ModelOutput`] types are:

            - [`~generation.GreedySearchEncoderDecoderOutput`],
            - [`~generation.SampleEncoderDecoderOutput`],
            - [`~generation.BeamSearchEncoderDecoderOutput`],
            - [`~generation.BeamSampleEncoderDecoderOutput`]
    """
    if generation_config is None:
        generation_config = self.generation_config

    if return_timestamps is not None:
        if not hasattr(generation_config, "no_timestamps_token_id"):
            raise ValueError(
                "You are trying to return timestamps, but the generation config is not properly set. "
                "Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. "
                "For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363"
            )

        generation_config.return_timestamps = return_timestamps
    else:
        generation_config.return_timestamps = False

    if is_multilingual is not None:
        if not hasattr(generation_config, "is_multilingual"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `is_multilingual` argument "
                "to `generate`. Please update the generation config as per the instructions "
                "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        generation_config.is_multilingual = is_multilingual

    if hasattr(generation_config, "is_multilingual") and not generation_config.is_multilingual:
        if task is not None or language is not None:
            raise ValueError(
                "Cannot specify `task` or `language` for an English-only model. If the model is intended to be "
                "multilingual, pass `is_multilingual=True` to generate, or update the generation config."
            )

    if language is not None:
        if not hasattr(generation_config, "lang_to_id"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `language` argument "
                "to `generate`. Either set the language using the `forced_decoder_ids` in the model config, "
                "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        language = language.lower()
        generation_config.language = language
    if task is not None:
        if not hasattr(generation_config, "task_to_id"):
            raise ValueError(
                "The generation config is outdated and is thus not compatible with the `task` argument "
                "to `generate`. Either set the task using the `forced_decoder_ids` in the model config, "
                "or update the generation config as per the instructions https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224"
            )
        generation_config.task = task

    forced_decoder_ids = None

    # Legacy code for backward compatibility
    if hasattr(self.config, "forced_decoder_ids") and self.config.forced_decoder_ids is not None:
        forced_decoder_ids = self.config.forced_decoder_ids
    elif (
        hasattr(self.generation_config, "forced_decoder_ids")
        and self.generation_config.forced_decoder_ids is not None
    ):
        forced_decoder_ids = self.generation_config.forced_decoder_ids
    else:
        forced_decoder_ids = kwargs.get("forced_decoder_ids", None)

    if task is not None or language is not None or (forced_decoder_ids is None and prompt_ids is not None):
        forced_decoder_ids = []
        if hasattr(generation_config, "language"):
            if generation_config.language in generation_config.lang_to_id.keys():
                language_token = generation_config.language
            elif generation_config.language in TO_LANGUAGE_CODE.keys():
                language_token = f"<|{TO_LANGUAGE_CODE[generation_config.language]}|>"
            elif generation_config.language in TO_LANGUAGE_CODE.values():
                language_token = f"<|{generation_config.language}|>"
            else:
                is_language_code = len(generation_config.language) == 2
                raise ValueError(
                    f"Unsupported language: {generation_config.language}. Language should be one of:"
                    f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}."
                )
            forced_decoder_ids.append((1, generation_config.lang_to_id[language_token]))
        else:
            forced_decoder_ids.append((1, None))  # automatically detect the language

        if hasattr(generation_config, "task"):
            if generation_config.task in TASK_IDS:
                forced_decoder_ids.append((2, generation_config.task_to_id[generation_config.task]))
            else:
                raise ValueError(
                    f"The `{generation_config.task}`task is not supported. The task should be one of `{TASK_IDS}`"
                )
        elif hasattr(generation_config, "task_to_id"):
            forced_decoder_ids.append((2, generation_config.task_to_id["transcribe"]))  # defaults to transcribe
        if hasattr(generation_config, "no_timestamps_token_id") and not generation_config.return_timestamps:
            idx = forced_decoder_ids[-1][0] + 1 if forced_decoder_ids else 1
            forced_decoder_ids.append((idx, generation_config.no_timestamps_token_id))

    if forced_decoder_ids is not None:
        generation_config.forced_decoder_ids = forced_decoder_ids

    if prompt_ids is not None:
        if kwargs.get("decoder_start_token_id") is not None:
            raise ValueError(
                "When specifying `prompt_ids`, you cannot also specify `decoder_start_token_id` as it gets overwritten."
            )
        prompt_ids = prompt_ids.tolist()
        decoder_start_token_id, *text_prompt_ids = prompt_ids
        # Slicing the text prompt ids in a manner consistent with the OpenAI implementation
        # to accomodate context space for the prefix (see https://github.com/openai/whisper/blob/c09a7ae299c4c34c5839a76380ae407e7d785914/whisper/decoding.py#L599)
        text_prompt_ids = text_prompt_ids[-self.config.max_target_positions // 2 - 1 :]
        # Set the decoder_start_token_id to <|startofprev|>
        kwargs.update({"decoder_start_token_id": decoder_start_token_id})

        # If the user passes `max_new_tokens`, increase its number to account for the prompt
        if kwargs.get("max_new_tokens", None) is not None:
            kwargs["max_new_tokens"] += len(text_prompt_ids)
            if kwargs["max_new_tokens"] >= self.config.max_target_positions:
                raise ValueError(
                    f"The length of the sliced `prompt_ids` is {len(text_prompt_ids)}, and the `max_new_tokens` "
                    f"{kwargs['max_new_tokens'] - len(text_prompt_ids)}. Thus, the combined length of the sliced "
                    f"`prompt_ids` and `max_new_tokens` is: {kwargs['max_new_tokens']}. This exceeds the "
                    f"`max_target_positions` of the Whisper model: {self.config.max_target_positions}. "
                    "You should either reduce the length of your prompt, or reduce the value of `max_new_tokens`, "
                    f"so that their combined length is less that {self.config.max_target_positions}."
                )

        # Reformat the forced_decoder_ids to incorporate the prompt
        non_prompt_forced_decoder_ids = (
            kwargs.pop("forced_decoder_ids", None) or generation_config.forced_decoder_ids
        )
        forced_decoder_ids = [
            *text_prompt_ids,
            generation_config.decoder_start_token_id,
            *[token for _rank, token in non_prompt_forced_decoder_ids],
        ]
        forced_decoder_ids = [(rank + 1, token) for rank, token in enumerate(forced_decoder_ids)]
        generation_config.forced_decoder_ids = forced_decoder_ids

    if generation_config.return_timestamps:
        logits_processor = [WhisperTimeStampLogitsProcessor(generation_config)]

    if return_token_timestamps:
        kwargs["output_attentions"] = True
        kwargs["return_dict_in_generate"] = True

        if getattr(generation_config, "task", None) == "translate":
            logger.warning("Token-level timestamps may not be reliable for task 'translate'.")
        if not hasattr(generation_config, "alignment_heads"):
            raise ValueError(
                "Model generation config has no `alignment_heads`, token-level timestamps not available. "
                "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config."
            )

        if kwargs.get("num_frames") is not None:
            generation_config.num_frames = kwargs.pop("num_frames")

    outputs = super().generate(
        inputs,
        generation_config,
        logits_processor,
        stopping_criteria,
        prefix_allowed_tokens_fn,
        synced_gpus,
        **kwargs,
    )

    if return_token_timestamps and hasattr(generation_config, "alignment_heads"):
        num_frames = getattr(generation_config, "num_frames", None)
        outputs["token_timestamps"] = self._extract_token_timestamps(
            outputs, generation_config.alignment_heads, num_frames=num_frames
        )

    return outputs

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_decoder()

This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves the decoder from the model.

PARAMETER	DESCRIPTION
self	Instance of the 'WhisperForConditionalGeneration' class. Type: object Purpose: Represents the current instance of the class. Restrictions: This parameter is required for accessing the decoder.

RETURNS	DESCRIPTION
None	Type: None Purpose: The method returns None as it retrieves the decoder from the model.

Source code in mindnlp/transformers/models/whisper/modeling_whisper.py

def get_decoder(self):
    """
    This method 'get_decoder' is part of the class 'WhisperForConditionalGeneration' and retrieves 
    the decoder from the model.

    Args:
        self:
            Instance of the 'WhisperForConditionalGeneration' class.

            - Type: object
            - Purpose: Represents the current instance of the class.
            - Restrictions: This parameter is required for accessing the decoder.

    Returns:
        None:

            - Type: None
            - Purpose: The method returns None as it retrieves the decoder from the model.

    Raises:
        None.
    """
    return self.model.get_decoder()

mindnlp.transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.get_encoder()

Retrieves the encoder from the model instance.

PARAMETER	DESCRIPTIO