speech_encoder_decoder

mindnlp.transformers.models.speech_encoder_decoder.configuration_speech_encoder_decoder

SpeechEncoderDecoder model configuration

mindnlp.transformers.models.speech_encoder_decoder.configuration_speech_encoder_decoder.SpeechEncoderDecoderConfig

Bases: PretrainedConfig

[SpeechEncoderDecoderConfig] is the configuration class to store the configuration of a [SpeechEncoderDecoderModel]. It is used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder configs.

Configuration objects inherit from [PretrainedConfig] and can be used to control the model outputs. Read the documentation from [PretrainedConfig] for more information.

PARAMETER	DESCRIPTION
kwargs	Dictionary of keyword arguments. Notably: encoder ([PretrainedConfig], optional) -- An instance of a configuration object that defines the encoder config. decoder ([PretrainedConfig], optional) -- An instance of a configuration object that defines the decoder config. TYPE: *optional* DEFAULT: {}

Example

>>> from transformers import BertConfig, Wav2Vec2Config, SpeechEncoderDecoderConfig, SpeechEncoderDecoderModel
...
>>> # Initializing a Wav2Vec2 & BERT style configuration
>>> config_encoder = Wav2Vec2Config()
>>> config_decoder = BertConfig()
...
>>> config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
...
>>> # Initializing a Wav2Vec2Bert model from a Wav2Vec2 & google-bert/bert-base-uncased style configurations
>>> model = SpeechEncoderDecoderModel(config=config)
...
>>> # Accessing the model configuration
>>> config_encoder = model.config.encoder
>>> config_decoder = model.config.decoder
>>> # set decoder config to causal lm
>>> config_decoder.is_decoder = True
>>> config_decoder.add_cross_attention = True
...
>>> # Saving the model, including its configuration
>>> model.save_pretrained("my-model")
...
>>> # loading model and config from pretrained folder
>>> encoder_decoder_config = SpeechEncoderDecoderConfig.from_pretrained("my-model")
>>> model = SpeechEncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config)

Source code in mindnlp/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py

class SpeechEncoderDecoderConfig(PretrainedConfig):
    r"""
    [`SpeechEncoderDecoderConfig`] is the configuration class to store the configuration of a
    [`SpeechEncoderDecoderModel`]. It is used to instantiate an Encoder Decoder model according to the specified
    arguments, defining the encoder and decoder configs.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        kwargs (*optional*):
            Dictionary of keyword arguments. Notably:

            - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
            the encoder config.
            - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
            the decoder config.

    Example:
        ```python
        >>> from transformers import BertConfig, Wav2Vec2Config, SpeechEncoderDecoderConfig, SpeechEncoderDecoderModel
        ...
        >>> # Initializing a Wav2Vec2 & BERT style configuration
        >>> config_encoder = Wav2Vec2Config()
        >>> config_decoder = BertConfig()
        ...
        >>> config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
        ...
        >>> # Initializing a Wav2Vec2Bert model from a Wav2Vec2 & google-bert/bert-base-uncased style configurations
        >>> model = SpeechEncoderDecoderModel(config=config)
        ...
        >>> # Accessing the model configuration
        >>> config_encoder = model.config.encoder
        >>> config_decoder = model.config.decoder
        >>> # set decoder config to causal lm
        >>> config_decoder.is_decoder = True
        >>> config_decoder.add_cross_attention = True
        ...
        >>> # Saving the model, including its configuration
        >>> model.save_pretrained("my-model")
        ...
        >>> # loading model and config from pretrained folder
        >>> encoder_decoder_config = SpeechEncoderDecoderConfig.from_pretrained("my-model")
        >>> model = SpeechEncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config)
        ```
    """

    model_type = "speech-encoder-decoder"
    is_composition = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        if "encoder" not in kwargs or "decoder" not in kwargs:
            raise ValueError(
                f"A configuraton of type {self.model_type} cannot be instantiated because not both `encoder` and"
                f" `decoder` sub-configurations are passed, but only {kwargs}"
            )

        encoder_config = kwargs.pop("encoder")
        encoder_model_type = encoder_config.pop("model_type")
        decoder_config = kwargs.pop("decoder")
        decoder_model_type = decoder_config.pop("model_type")

        self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
        self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
        self.is_encoder_decoder = True

    @classmethod
    def from_encoder_decoder_configs(
        cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
    ) -> PretrainedConfig:
        r"""
        Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
        configuration and decoder model configuration.

        Returns:
            [`SpeechEncoderDecoderConfig`]: An instance of a configuration object
        """
        logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
        decoder_config.is_decoder = True
        decoder_config.add_cross_attention = True

        return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

mindnlp.transformers.models.speech_encoder_decoder.configuration_speech_encoder_decoder.SpeechEncoderDecoderConfig.from_encoder_decoder_configs(encoder_config, decoder_config, **kwargs) classmethod

Instantiate a [SpeechEncoderDecoderConfig] (or a derived class) from a pre-trained encoder model configuration and decoder model configuration.

RETURNS	DESCRIPTION
PretrainedConfig	[SpeechEncoderDecoderConfig]: An instance of a configuration object

Source code in mindnlp/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py

@classmethod
def from_encoder_decoder_configs(
    cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
) -> PretrainedConfig:
    r"""
    Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
    configuration and decoder model configuration.

    Returns:
        [`SpeechEncoderDecoderConfig`]: An instance of a configuration object
    """
    logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
    decoder_config.is_decoder = True
    decoder_config.add_cross_attention = True

    return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder

Classes to support Speech-Encoder-Text-Decoder architectures

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder.SpeechEncoderDecoderModel

Bases: PreTrainedModel

[SpeechEncoderDecoderModel] is a generic model class that will be instantiated as a transformer architecture with one of the base model classes of the library as encoder and another one as decoder when created with the :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and :meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.

Source code in mindnlp/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py

class SpeechEncoderDecoderModel(PreTrainedModel):
    r"""
    [`SpeechEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with
    one of the base model classes of the library as encoder and another one as decoder when created with the
    :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
    :meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.
    """

    config_class = SpeechEncoderDecoderConfig
    base_model_prefix = "speech_encoder_decoder"
    main_input_name = "inputs"
    supports_gradient_checkpointing = True

    def __init__(
        self,
        config: Optional[PretrainedConfig] = None,
        encoder: Optional[PreTrainedModel] = None,
        decoder: Optional[PreTrainedModel] = None,
    ):
        if config is None and (encoder is None or decoder is None):
            raise ValueError("Either a configuration or an encoder and a decoder has to be provided.")
        if config is None:
            config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config)
        else:
            if not isinstance(config, self.config_class):
                raise ValueError(f"Config: {config} has to be of type {self.config_class}")

        if config.decoder.cross_attention_hidden_size is not None:
            if config.decoder.cross_attention_hidden_size != config.encoder.hidden_size:
                raise ValueError(
                    "If `cross_attention_hidden_size` is specified in the decoder's configuration, it has to be equal"
                    f" to the encoder's `hidden_size`. Got {config.decoder.cross_attention_hidden_size} for"
                    f" `config.decoder.cross_attention_hidden_size` and {config.encoder.hidden_size} for"
                    " `config.encoder.hidden_size`."
                )

        # initialize with config
        # make sure input & output embeddings is not tied
        config.tie_word_embeddings = False
        super().__init__(config)

        if encoder is None:
            encoder = AutoModel.from_config(config.encoder)

        if decoder is None:
            decoder = AutoModelForCausalLM.from_config(config.decoder)

        self.encoder = encoder
        self.decoder = decoder

        if self.encoder.config.to_dict() != self.config.encoder.to_dict():
            logger.warning(
                f"Config of the encoder: {self.encoder.__class__} is overwritten by shared encoder config:"
                f" {self.config.encoder}"
            )
        if self.decoder.config.to_dict() != self.config.decoder.to_dict():
            logger.warning(
                f"Config of the decoder: {self.decoder.__class__} is overwritten by shared decoder config:"
                f" {self.config.decoder}"
            )

        # make sure that the individual model's config refers to the shared config
        # so that the updates to the config will be synced
        self.encoder.config = self.config.encoder
        self.decoder.config = self.config.decoder

        # get encoder output hidden size
        self.encoder_output_dim = getattr(config.encoder, "output_hidden_size", config.encoder.hidden_size)
        if (
            self.encoder_output_dim != self.decoder.config.hidden_size
            and self.decoder.config.cross_attention_hidden_size is None
        ):
            # encoder outputs might need to be projected to different dimension for decoder
            self.enc_to_dec_proj = nn.Linear(self.encoder.config.hidden_size, self.decoder.config.hidden_size)

        if self.encoder.get_output_embeddings() is not None:
            raise ValueError(
                f"The encoder {self.encoder} should not have a LM Head. Please use a model without LM Head"
            )

    def get_encoder(self):
        return self.encoder

    def get_decoder(self):
        return self.decoder

    def get_output_embeddings(self):
        return self.decoder.get_output_embeddings()

    def set_output_embeddings(self, new_embeddings):
        return self.decoder.set_output_embeddings(new_embeddings)

    def freeze_feature_encoder(self):
        """
        Calling this function will disable the gradient computation for the feature encoder of the speech encoder so
        that its parameters will not be updated during training.
        """
        self.encoder.freeze_feature_encoder()

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        # At the moment fast initialization is not supported for composite models
        if kwargs.get("_fast_init", False):
            logger.warning(
                "Fast initialization is currently not supported for SpeechEncoderDecoderModel. "
                "Falling back to slow initialization..."
            )
        kwargs["_fast_init"] = False
        return super().from_pretrained(*args, **kwargs)

    @classmethod
    def from_encoder_decoder_pretrained(
        cls,
        *model_args,
        encoder_pretrained_model_name_or_path: str = None,
        decoder_pretrained_model_name_or_path: str = None,
        **kwargs,
    ) -> PreTrainedModel:
        r"""
        Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
        checkpoints.

        The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
        the model, you need to first set it back in training mode with `model.train()`.

        Params:
            encoder_pretrained_model_name_or_path (`str`, *optional*):
                Information necessary to initiate the encoder. Can be either:

                - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
                - A path to a *directory* containing model weights saved using
                [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
                - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
                this case, `from_tf` should be set to `True` and a configuration object should be provided as
                `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
                PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.

            decoder_pretrained_model_name_or_path (`str`, *optional*, defaults to `None`):
                Information necessary to initiate the decoder. Can be either:

                - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
                - A path to a *directory* containing model weights saved using
                [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
                - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
                this case, `from_tf` should be set to `True` and a configuration object should be provided as
                `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
                PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.

            model_args (remaining positional arguments, *optional*):
                All remaning positional arguments will be passed to the underlying model's `__init__` method.

            kwargs (remaining dictionary of keyword arguments, *optional*):
                Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
                `output_attentions=True`).

                - To update the encoder configuration, use the prefix *encoder_* for each configuration parameter.
                - To update the decoder configuration, use the prefix *decoder_* for each configuration parameter.
                - To update the parent model configuration, do not use a prefix for each configuration parameter.

                Behaves differently depending on whether a `config` is provided or automatically loaded.

        Example:
            ```python
            >>> from transformers import SpeechEncoderDecoderModel
            ...
            >>> # initialize a wav2vec2bert from a pretrained Wav2Vec2 and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
            >>> model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
            ...     "facebook/wav2vec2-base-960h", "google-bert/bert-base-uncased"
            ... )
            >>> # saving model after fine-tuning
            >>> model.save_pretrained("./wav2vec2bert")
            >>> # load fine-tuned model
            >>> model = SpeechEncoderDecoderModel.from_pretrained("./wav2vec2bert")
            ```
        """

        kwargs_encoder = {
            argument[len("encoder_") :]: value for argument, value in kwargs.items() if argument.startswith("encoder_")
        }

        kwargs_decoder = {
            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
        }

        # remove encoder, decoder kwargs from kwargs
        for key in kwargs_encoder.keys():
            del kwargs["encoder_" + key]
        for key in kwargs_decoder.keys():
            del kwargs["decoder_" + key]

        # Load and initialize the encoder and decoder
        # The distinction between encoder and decoder at the model level is made
        # by the value of the flag `is_decoder` that we need to set correctly.
        encoder = kwargs_encoder.pop("model", None)
        if encoder is None:
            if encoder_pretrained_model_name_or_path is None:
                raise ValueError(
                    "If `encoder_model` is not defined as an argument, a `encoder_pretrained_model_name_or_path` has "
                    "to be defined."
                )

            if "config" not in kwargs_encoder:
                encoder_config, kwargs_encoder = AutoConfig.from_pretrained(
                    encoder_pretrained_model_name_or_path, **kwargs_encoder, return_unused_kwargs=True
                )

                if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:
                    logger.info(
                        f"Initializing {encoder_pretrained_model_name_or_path} as a encoder model "
                        "from a decoder model. Cross-attention and casual mask are disabled."
                    )
                    encoder_config.is_decoder = False
                    encoder_config.add_cross_attention = False

                kwargs_encoder["config"] = encoder_config

            encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)

        decoder = kwargs_decoder.pop("model", None)
        if decoder is None:
            if decoder_pretrained_model_name_or_path is None:
                raise ValueError(
                    "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has "
                    "to be defined."
                )

            if "config" not in kwargs_decoder:
                decoder_config, kwargs_decoder = AutoConfig.from_pretrained(
                    decoder_pretrained_model_name_or_path, **kwargs_decoder, return_unused_kwargs=True
                )

                if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:
                    logger.info(
                        f"Initializing {decoder_pretrained_model_name_or_path} as a decoder model. Cross attention"
                        f" layers are added to {decoder_pretrained_model_name_or_path} and randomly initialized if"
                        f" {decoder_pretrained_model_name_or_path}'s architecture allows for cross attention layers."
                    )
                    decoder_config.is_decoder = True
                    decoder_config.add_cross_attention = True

                kwargs_decoder["config"] = decoder_config

            if kwargs_decoder["config"].is_decoder is False or kwargs_decoder["config"].add_cross_attention is False:
                logger.warning(
                    f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. "
                    f"In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, "
                    "make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` "
                    "passed to `.from_encoder_decoder_pretrained(...)` are set to `True` or do not pass a "
                    "`decoder_config` to `.from_encoder_decoder_pretrained(...)`"
                )

            decoder = AutoModelForCausalLM.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)

        # instantiate config with corresponding kwargs
        config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config, **kwargs)

        # make sure input & output embeddings is not tied
        config.tie_word_embeddings = False
        return cls(encoder=encoder, decoder=decoder, config=config)

    # @add_start_docstrings_to_model_forward(SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING)
    # @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        inputs: Optional[mindspore.Tensor] = None,
        attention_mask: Optional[mindspore.Tensor] = None,
        decoder_input_ids: Optional[mindspore.Tensor] = None,
        decoder_attention_mask: Optional[mindspore.Tensor] = None,
        encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
        past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
        decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
        labels: Optional[mindspore.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        input_values: Optional[mindspore.Tensor] = None,
        input_features: Optional[mindspore.Tensor] = None,
        return_dict: Optional[bool] = None,
        **kwargs,
    ) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
        r"""
        Returns:
            `Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]`

        Example:
            ```python
            >>> from transformers import SpeechEncoderDecoderModel, AutoProcessor
            >>> from datasets import load_dataset
            >>> import torch
            ...
            >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
            >>> model = SpeechEncoderDecoderModel.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
            ...
            >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
            ...
            >>> input_values = processor(ds[0]["audio"]["array"], return_tensors="pt").input_values
            >>> # Inference: Translate English speech to German
            >>> generated = model.generate(input_values)
            >>> decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
            >>> decoded
            'Mr. Quilter ist der Apostel der Mittelschicht und wir freuen uns, sein Evangelium willkommen heißen zu können.'
            >>> # Training: Train model on English transcription
            >>> labels = processor(text=ds[0]["text"], return_tensors="pt").input_ids
            ...
            >>> loss = model(input_values, labels=labels).loss
            >>> loss.backward()
            ```
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")}

        kwargs_decoder = {
            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
        }

        if encoder_outputs is None:
            if inputs is None:
                if input_values is not None and input_features is not None:
                    raise ValueError("You cannot specify both input_values and input_features at the same time")
                elif input_values is not None:
                    inputs = input_values
                elif input_features is not None:
                    inputs = input_features
                else:
                    raise ValueError("You have to specify either input_values or input_features")

            encoder_outputs = self.encoder(
                inputs,
                attention_mask=attention_mask,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                **kwargs_encoder,
            )
        elif isinstance(encoder_outputs, tuple):
            encoder_outputs = BaseModelOutput(*encoder_outputs)

        encoder_hidden_states = encoder_outputs[0]

        # optionally project encoder_hidden_states
        if (
            self.encoder_output_dim != self.decoder.config.hidden_size
            and self.decoder.config.cross_attention_hidden_size is None
        ):
            encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states)

        # compute correct encoder attention mask
        if attention_mask is not None:
            encoder_attention_mask = self.encoder._get_feature_vector_attention_mask(
                encoder_hidden_states.shape[1], attention_mask
            )
        else:
            encoder_attention_mask = None

        if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None):
            decoder_input_ids = shift_tokens_right(
                labels, self.config.pad_token_id, self.config.decoder_start_token_id
            )

        # Decode
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            inputs_embeds=decoder_inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            use_cache=use_cache,
            past_key_values=past_key_values,
            return_dict=return_dict,
            **kwargs_decoder,
        )

        # Compute loss independent from decoder (as some shift the logits inside them)
        loss = None
        if labels is not None:
            logits = decoder_outputs.logits if return_dict else decoder_outputs[0]
            loss = ops.cross_entropy(logits.reshape(-1, self.decoder.config.vocab_size), labels.reshape(-1))

        if not return_dict:
            if loss is not None:
                return (loss,) + decoder_outputs + encoder_outputs
            else:
                return decoder_outputs + encoder_outputs

        return Seq2SeqLMOutput(
            loss=loss,
            logits=decoder_outputs.logits,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_hidden_states,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )

    def prepare_decoder_input_ids_from_labels(self, labels: mindspore.Tensor):
        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)

    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
    ):
        decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids, past_key_values=past_key_values)
        decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None
        input_dict = {
            "attention_mask": attention_mask,
            "decoder_attention_mask": decoder_attention_mask,
            "decoder_input_ids": decoder_inputs["input_ids"],
            "encoder_outputs": encoder_outputs,
            "past_key_values": decoder_inputs["past_key_values"],
            "use_cache": use_cache,
        }
        return input_dict

    def resize_token_embeddings(self, *args, **kwargs):
        raise NotImplementedError(
            "Resizing the embedding layers via the SpeechEncoderDecoderModel directly is not supported. Please use the"
            " respective methods of the wrapped decoder object (model.decoder.resize_token_embeddings(...))"
        )

    def _reorder_cache(self, past_key_values, beam_idx):
        # apply decoder cache reordering here
        return self.decoder._reorder_cache(past_key_values, beam_idx)

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder.SpeechEncoderDecoderModel.forward(inputs=None, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, encoder_outputs=None, past_key_values=None, decoder_inputs_embeds=None, labels=None, use_cache=None, output_attentions=None, output_hidden_states=None, input_values=None, input_features=None, return_dict=None, **kwargs)

RETURNS	DESCRIPTION
Union[Tuple[Tensor], Seq2SeqLMOutput]	Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]

Example

>>> from transformers import SpeechEncoderDecoderModel, AutoProcessor
>>> from datasets import load_dataset
>>> import torch
...
>>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
>>> model = SpeechEncoderDecoderModel.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
...
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
...
>>> input_values = processor(ds[0]["audio"]["array"], return_tensors="pt").input_values
>>> # Inference: Translate English speech to German
>>> generated = model.generate(input_values)
>>> decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
>>> decoded
'Mr. Quilter ist der Apostel der Mittelschicht und wir freuen uns, sein Evangelium willkommen heißen zu können.'
>>> # Training: Train model on English transcription
>>> labels = processor(text=ds[0]["text"], return_tensors="pt").input_ids
...
>>> loss = model(input_values, labels=labels).loss
>>> loss.backward()

Source code in mindnlp/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py

def forward(
    self,
    inputs: Optional[mindspore.Tensor] = None,
    attention_mask: Optional[mindspore.Tensor] = None,
    decoder_input_ids: Optional[mindspore.Tensor] = None,
    decoder_attention_mask: Optional[mindspore.Tensor] = None,
    encoder_outputs: Optional[Tuple[mindspore.Tensor]] = None,
    past_key_values: Optional[Tuple[Tuple[mindspore.Tensor]]] = None,
    decoder_inputs_embeds: Optional[mindspore.Tensor] = None,
    labels: Optional[mindspore.Tensor] = None,
    use_cache: Optional[bool] = None,
    output_attentions: Optional[bool] = None,
    output_hidden_states: Optional[bool] = None,
    input_values: Optional[mindspore.Tensor] = None,
    input_features: Optional[mindspore.Tensor] = None,
    return_dict: Optional[bool] = None,
    **kwargs,
) -> Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]:
    r"""
    Returns:
        `Union[Tuple[mindspore.Tensor], Seq2SeqLMOutput]`

    Example:
        ```python
        >>> from transformers import SpeechEncoderDecoderModel, AutoProcessor
        >>> from datasets import load_dataset
        >>> import torch
        ...
        >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
        >>> model = SpeechEncoderDecoderModel.from_pretrained("facebook/wav2vec2-xls-r-300m-en-to-15")
        ...
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        ...
        >>> input_values = processor(ds[0]["audio"]["array"], return_tensors="pt").input_values
        >>> # Inference: Translate English speech to German
        >>> generated = model.generate(input_values)
        >>> decoded = processor.batch_decode(generated, skip_special_tokens=True)[0]
        >>> decoded
        'Mr. Quilter ist der Apostel der Mittelschicht und wir freuen uns, sein Evangelium willkommen heißen zu können.'
        >>> # Training: Train model on English transcription
        >>> labels = processor(text=ds[0]["text"], return_tensors="pt").input_ids
        ...
        >>> loss = model(input_values, labels=labels).loss
        >>> loss.backward()
        ```
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")}

    kwargs_decoder = {
        argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
    }

    if encoder_outputs is None:
        if inputs is None:
            if input_values is not None and input_features is not None:
                raise ValueError("You cannot specify both input_values and input_features at the same time")
            elif input_values is not None:
                inputs = input_values
            elif input_features is not None:
                inputs = input_features
            else:
                raise ValueError("You have to specify either input_values or input_features")

        encoder_outputs = self.encoder(
            inputs,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            **kwargs_encoder,
        )
    elif isinstance(encoder_outputs, tuple):
        encoder_outputs = BaseModelOutput(*encoder_outputs)

    encoder_hidden_states = encoder_outputs[0]

    # optionally project encoder_hidden_states
    if (
        self.encoder_output_dim != self.decoder.config.hidden_size
        and self.decoder.config.cross_attention_hidden_size is None
    ):
        encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states)

    # compute correct encoder attention mask
    if attention_mask is not None:
        encoder_attention_mask = self.encoder._get_feature_vector_attention_mask(
            encoder_hidden_states.shape[1], attention_mask
        )
    else:
        encoder_attention_mask = None

    if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None):
        decoder_input_ids = shift_tokens_right(
            labels, self.config.pad_token_id, self.config.decoder_start_token_id
        )

    # Decode
    decoder_outputs = self.decoder(
        input_ids=decoder_input_ids,
        attention_mask=decoder_attention_mask,
        encoder_hidden_states=encoder_hidden_states,
        encoder_attention_mask=encoder_attention_mask,
        inputs_embeds=decoder_inputs_embeds,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        use_cache=use_cache,
        past_key_values=past_key_values,
        return_dict=return_dict,
        **kwargs_decoder,
    )

    # Compute loss independent from decoder (as some shift the logits inside them)
    loss = None
    if labels is not None:
        logits = decoder_outputs.logits if return_dict else decoder_outputs[0]
        loss = ops.cross_entropy(logits.reshape(-1, self.decoder.config.vocab_size), labels.reshape(-1))

    if not return_dict:
        if loss is not None:
            return (loss,) + decoder_outputs + encoder_outputs
        else:
            return decoder_outputs + encoder_outputs

    return Seq2SeqLMOutput(
        loss=loss,
        logits=decoder_outputs.logits,
        past_key_values=decoder_outputs.past_key_values,
        decoder_hidden_states=decoder_outputs.hidden_states,
        decoder_attentions=decoder_outputs.attentions,
        cross_attentions=decoder_outputs.cross_attentions,
        encoder_last_hidden_state=encoder_hidden_states,
        encoder_hidden_states=encoder_outputs.hidden_states,
        encoder_attentions=encoder_outputs.attentions,
    )

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder.SpeechEncoderDecoderModel.freeze_feature_encoder()

Calling this function will disable the gradient computation for the feature encoder of the speech encoder so that its parameters will not be updated during training.

Source code in mindnlp/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py

def freeze_feature_encoder(self):
    """
    Calling this function will disable the gradient computation for the feature encoder of the speech encoder so
    that its parameters will not be updated during training.
    """
    self.encoder.freeze_feature_encoder()

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(*model_args, encoder_pretrained_model_name_or_path=None, decoder_pretrained_model_name_or_path=None, **kwargs) classmethod

Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model checkpoints.

The model is set in evaluation mode by default using model.eval() (Dropout modules are deactivated). To train the model, you need to first set it back in training mode with model.train().

PARAMETER	DESCRIPTION
encoder_pretrained_model_name_or_path	Information necessary to initiate the encoder. Can be either: A string, the model id of a pretrained model hosted inside a model repo on huggingface.co. A path to a directory containing model weights saved using [~PreTrainedModel.save_pretrained], e.g., ./my_model_directory/. A path or url to a tensorflow index checkpoint file (e.g, ./tf_model/model.ckpt.index). In this case, from_tf should be set to True and a configuration object should be provided as config argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. TYPE: `str`, *optional* DEFAULT: None
decoder_pretrained_model_name_or_path	Information necessary to initiate the decoder. Can be either: A string, the model id of a pretrained model hosted inside a model repo on huggingface.co. A path to a directory containing model weights saved using [~PreTrainedModel.save_pretrained], e.g., ./my_model_directory/. A path or url to a tensorflow index checkpoint file (e.g, ./tf_model/model.ckpt.index). In this case, from_tf should be set to True and a configuration object should be provided as config argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. TYPE: `str`, *optional*, defaults to `None` DEFAULT: None
model_args	All remaning positional arguments will be passed to the underlying model's __init__ method. TYPE: remaining positional arguments, *optional* DEFAULT: ()
kwargs	Can be used to update the configuration object (after it being loaded) and initiate the model (e.g., output_attentions=True). To update the encoder configuration, use the prefix encoder_ for each configuration parameter. To update the decoder configuration, use the prefix decoder_ for each configuration parameter. To update the parent model configuration, do not use a prefix for each configuration parameter. Behaves differently depending on whether a config is provided or automatically loaded. TYPE: remaining dictionary of keyword arguments, *optional* DEFAULT: {}

Example

>>> from transformers import SpeechEncoderDecoderModel
...
>>> # initialize a wav2vec2bert from a pretrained Wav2Vec2 and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
>>> model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
...     "facebook/wav2vec2-base-960h", "google-bert/bert-base-uncased"
... )
>>> # saving model after fine-tuning
>>> model.save_pretrained("./wav2vec2bert")
>>> # load fine-tuned model
>>> model = SpeechEncoderDecoderModel.from_pretrained("./wav2vec2bert")

Source code in mindnlp/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py

@classmethod
def from_encoder_decoder_pretrained(
    cls,
    *model_args,
    encoder_pretrained_model_name_or_path: str = None,
    decoder_pretrained_model_name_or_path: str = None,
    **kwargs,
) -> PreTrainedModel:
    r"""
    Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
    checkpoints.

    The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
    the model, you need to first set it back in training mode with `model.train()`.

    Params:
        encoder_pretrained_model_name_or_path (`str`, *optional*):
            Information necessary to initiate the encoder. Can be either:

            - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
            - A path to a *directory* containing model weights saved using
            [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
            - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
            this case, `from_tf` should be set to `True` and a configuration object should be provided as
            `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
            PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.

        decoder_pretrained_model_name_or_path (`str`, *optional*, defaults to `None`):
            Information necessary to initiate the decoder. Can be either:

            - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
            - A path to a *directory* containing model weights saved using
            [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
            - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
            this case, `from_tf` should be set to `True` and a configuration object should be provided as
            `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
            PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.

        model_args (remaining positional arguments, *optional*):
            All remaning positional arguments will be passed to the underlying model's `__init__` method.

        kwargs (remaining dictionary of keyword arguments, *optional*):
            Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
            `output_attentions=True`).

            - To update the encoder configuration, use the prefix *encoder_* for each configuration parameter.
            - To update the decoder configuration, use the prefix *decoder_* for each configuration parameter.
            - To update the parent model configuration, do not use a prefix for each configuration parameter.

            Behaves differently depending on whether a `config` is provided or automatically loaded.

    Example:
        ```python
        >>> from transformers import SpeechEncoderDecoderModel
        ...
        >>> # initialize a wav2vec2bert from a pretrained Wav2Vec2 and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
        >>> model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
        ...     "facebook/wav2vec2-base-960h", "google-bert/bert-base-uncased"
        ... )
        >>> # saving model after fine-tuning
        >>> model.save_pretrained("./wav2vec2bert")
        >>> # load fine-tuned model
        >>> model = SpeechEncoderDecoderModel.from_pretrained("./wav2vec2bert")
        ```
    """

    kwargs_encoder = {
        argument[len("encoder_") :]: value for argument, value in kwargs.items() if argument.startswith("encoder_")
    }

    kwargs_decoder = {
        argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
    }

    # remove encoder, decoder kwargs from kwargs
    for key in kwargs_encoder.keys():
        del kwargs["encoder_" + key]
    for key in kwargs_decoder.keys():
        del kwargs["decoder_" + key]

    # Load and initialize the encoder and decoder
    # The distinction between encoder and decoder at the model level is made
    # by the value of the flag `is_decoder` that we need to set correctly.
    encoder = kwargs_encoder.pop("model", None)
    if encoder is None:
        if encoder_pretrained_model_name_or_path is None:
            raise ValueError(
                "If `encoder_model` is not defined as an argument, a `encoder_pretrained_model_name_or_path` has "
                "to be defined."
            )

        if "config" not in kwargs_encoder:
            encoder_config, kwargs_encoder = AutoConfig.from_pretrained(
                encoder_pretrained_model_name_or_path, **kwargs_encoder, return_unused_kwargs=True
            )

            if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:
                logger.info(
                    f"Initializing {encoder_pretrained_model_name_or_path} as a encoder model "
                    "from a decoder model. Cross-attention and casual mask are disabled."
                )
                encoder_config.is_decoder = False
                encoder_config.add_cross_attention = False

            kwargs_encoder["config"] = encoder_config

        encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)

    decoder = kwargs_decoder.pop("model", None)
    if decoder is None:
        if decoder_pretrained_model_name_or_path is None:
            raise ValueError(
                "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has "
                "to be defined."
            )

        if "config" not in kwargs_decoder:
            decoder_config, kwargs_decoder = AutoConfig.from_pretrained(
                decoder_pretrained_model_name_or_path, **kwargs_decoder, return_unused_kwargs=True
            )

            if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:
                logger.info(
                    f"Initializing {decoder_pretrained_model_name_or_path} as a decoder model. Cross attention"
                    f" layers are added to {decoder_pretrained_model_name_or_path} and randomly initialized if"
                    f" {decoder_pretrained_model_name_or_path}'s architecture allows for cross attention layers."
                )
                decoder_config.is_decoder = True
                decoder_config.add_cross_attention = True

            kwargs_decoder["config"] = decoder_config

        if kwargs_decoder["config"].is_decoder is False or kwargs_decoder["config"].add_cross_attention is False:
            logger.warning(
                f"Decoder model {decoder_pretrained_model_name_or_path} is not initialized as a decoder. "
                f"In order to initialize {decoder_pretrained_model_name_or_path} as a decoder, "
                "make sure that the attributes `is_decoder` and `add_cross_attention` of `decoder_config` "
                "passed to `.from_encoder_decoder_pretrained(...)` are set to `True` or do not pass a "
                "`decoder_config` to `.from_encoder_decoder_pretrained(...)`"
            )

        decoder = AutoModelForCausalLM.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)

    # instantiate config with corresponding kwargs
    config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config, **kwargs)

    # make sure input & output embeddings is not tied
    config.tie_word_embeddings = False
    return cls(encoder=encoder, decoder=decoder, config=config)

mindnlp.transformers.models.speech_encoder_decoder.modeling_speech_encoder_decoder.shift_tokens_right(input_ids, pad_token_id, decoder_start_token_id)

Shift input ids one token to the right.

Source code in mindnlp/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py

def shift_tokens_right(input_ids: mindspore.Tensor, pad_token_id: int, decoder_start_token_id: int):
    """
    Shift input ids one token to the right.
    """
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    if decoder_start_token_id is None:
        raise ValueError("Make sure to set the decoder_start_token_id attribute of the model's configuration.")
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("Make sure to set the pad_token_id attribute of the model's configuration.")
    # replace possible -100 values in labels by `pad_token_id`
    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids