Skip to content

peft_model

mindnlp.peft.peft_model

PEFT model.

mindnlp.peft.peft_model.PeftModel

Bases: Module

Base model encompassing various Peft methods.

PARAMETER DESCRIPTION
model

The base transformer model used for Peft.

TYPE: [`~mindnlp.models.PreTrainedModel`]

peft_config

The configuration of the Peft model.

TYPE: [`PeftConfig`]

Source code in mindnlp/peft/peft_model.py 
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
class PeftModel(nn.Module):
    """
    Base model encompassing various Peft methods.

    Args:
        model ([`~mindnlp.models.PreTrainedModel`]): The base transformer model used for Peft.
        peft_config ([`PeftConfig`]): The configuration of the Peft model.
    """
    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
        r"""
        __init__

        This method initializes an instance of the PeftModel class.

        Args:
            self: The instance of the PeftModel class.
            model: The base model used for the PeftModel instance.
            peft_config (PeftConfig): An instance of PeftConfig class containing configuration for the PEFT (Prompt-based Entity Fine-Tuning) process.
            adapter_name (str, optional): The name of the adapter being used. Defaults to 'default'.

        Returns:
            None. This method does not return any value.

        Raises:
            - TypeError: If the provided model is not of the expected type.
            - ValueError: If the provided peft_config is not valid or does not contain necessary information.
            - KeyError: If there is an issue with accessing or setting attributes.
        """
        super().__init__()
        self.base_model = model
        self.config = getattr(self.base_model, "config", {"model_type": "custom"})
        self.cells_to_save = None
        self.peft_config: Dict[str, PeftConfig] = {}
        self.active_adapter = adapter_name
        self.peft_type = peft_config.peft_type
        self.base_model_dtype = getattr(model, "dtype", None)
        self.special_peft_forward_args = {"adapter_name"}
        if not peft_config.is_prompt_learning:
            self.peft_config[adapter_name] = peft_config
            self.base_model = PEFT_TYPE_TO_MODEL_MAPPING[peft_config.peft_type](
                self.base_model, self.peft_config, adapter_name
            )
            self.set_additional_trainable_cells(peft_config, adapter_name)
        else:
            self.add_adapter(adapter_name, peft_config)

        # if getattr(model, "is_gradient_checkpointing", True):
        #     model = self._prepare_model_for_gradient_checkpointing(model)
        # if hasattr(self.base_model, "config") and hasattr(self.base_model.config, "pretraining_tp"):
        #     self.base_model.config.pretraining_tp = 1

    def save_pretrained(self, save_directory, **kwargs):
        r"""
        This function saves the adapter model and the adapter configuration files to a directory, so that it can be
        reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`]
        method.
        """
        if os.path.isfile(save_directory):
            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
        os.makedirs(save_directory, exist_ok=True)

        for adapter_name, peft_config in self.peft_config.items():
            # save only the trainable weights
            output_state_dict = get_peft_model_state_dict(
                self,
                state_dict=kwargs.get("state_dict", None),
                adapter_name=adapter_name
            )
            output_dir = os.path.join(save_directory, adapter_name) if adapter_name != "default" else save_directory
            os.makedirs(output_dir, exist_ok=True)

            _exec_save(
                ckpt_file_name=os.path.join(output_dir, WEIGHTS_NAME),
                data_list=output_state_dict,
            )

            # save the config and change the inference mode to `True`
            if peft_config.base_model_name_or_path is None:
                peft_config.base_model_name_or_path = (
                    self.base_model.__dict__.get("name_or_path", None),
                    self.base_model.model.__dict__.get("name_or_path", None)
                )
            inference_mode = peft_config.inference_mode
            peft_config.inference_mode = True
            peft_config.save_pretrained(output_dir)
            peft_config.inference_mode = inference_mode

    @classmethod
    def from_pretrained(cls, model, model_id, adapter_name="default", is_trainable=False, **kwargs):
        r"""
        Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights.

        Args:
            model ([`~transformers.PreTrainedModel`]):
                The model to be adapted. The model should be initialized with the
                [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library.
            model_id (`str` or `os.PathLike`):
                The name of the Lora configuration to use. Can be :
                    - A path to a directory containing a Lora configuration file saved using the `save_pretrained`
                      method (`./my_lora_config_directory/`).
        """
        from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING
        # load peft config
        config = PEFT_TYPE_TO_CONFIG_MAPPING[
            PeftConfig.from_pretrained(model_id, subfolder=kwargs.get("subfolder", None)).peft_type
        ].from_pretrained(model_id, subfolder=kwargs.get("subfolder", None))

        config.inference_mode = not is_trainable

        if config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING:
            model = cls(model, config, adapter_name)
        else:
            model = MODEL_TYPE_TO_PEFT_MODEL_MAPPING[config.task_type](model, config, adapter_name)
        model.load_adapter(model_id, adapter_name, **kwargs)
        return model

    def _setup_prompt_encoder(self, adapter_name: str):
        r"""
        This method '_setup_prompt_encoder' in the class 'PeftModel' is responsible for setting up the prompt encoder based on the provided adapter name.

        Args:
        - self: The instance of the 'PeftModel' class.
        - adapter_name (str): The name of the adapter for which the prompt encoder is being set up. It is used to fetch configuration settings related to the specified adapter.

        Returns:
        None. This method does not return any value.

        Raises:
        - ValueError: Raised when the provided 'peft_type' in the configuration is not supported by the method.
        """
        config = self.peft_config[adapter_name]
        if not hasattr(self, "prompt_encoder"):
            self.prompt_encoder = CellDict({})
            self.prompt_tokens = {}
        transformer_backbone = None
        for name, cell in self.base_model.cells_and_names():
            for param in cell.get_parameters():
                param.requires_grad = False
            if isinstance(cell, PreTrainedModel):
                # Make sure to freeze Tranformers model
                if transformer_backbone is None:
                    transformer_backbone = cell
                    self.transformer_backbone_name = name
        if transformer_backbone is None:
            transformer_backbone = self.base_model

        if config.num_transformer_submodules is None:
            config.num_transformer_submodules = 2 if config.task_type == TaskType.SEQ_2_SEQ_LM else 1

        for named_param, value in list(transformer_backbone.parameters_and_names()):

            if value.shape[0] == self.base_model.config.vocab_size:
                self.word_embeddings = transformer_backbone.get_cell(named_param.replace(".weight", ""))
                break

        if config.peft_type == PeftType.PROMPT_TUNING:
            prompt_encoder = PromptEmbedding(config, self.word_embeddings)
        elif config.peft_type == PeftType.MULTITASK_PROMPT_TUNING:
            prompt_encoder = MultitaskPromptEmbedding(config, self.word_embeddings)
        elif config.peft_type == PeftType.P_TUNING:
            prompt_encoder = PromptEncoder(config)
        elif config.peft_type == PeftType.PREFIX_TUNING:
            prompt_encoder = PrefixEncoder(config)
        else:
            raise ValueError("Not supported")

        self.prompt_encoder.update(CellDict({adapter_name: prompt_encoder}))
        self.prompt_tokens[adapter_name] = ops.arange(
            config.num_virtual_tokens * config.num_transformer_submodules
        ).long()

    def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = False, **kwargs):
        """load adapter to peft model, called by `model.from_pretrained`."""
        # NOTE: remove download logic.
        if adapter_name not in self.peft_config:
            raise ValueError(f"{adapter_name} is not a valid adapter name. Valid names: {self.peft_config.keys()}")

        adapters_weights = load_peft_weights(model_id)

        # load the weights into the model
        load_result = set_peft_model_state_dict(self, adapters_weights, adapter_name=adapter_name)
        # TODO: add parallel logic & offload logic & device map logic(dispatch_model)

        # Set model in evaluation mode to deactivate Dropout cells by default
        if not is_trainable:
            self.set_train(False)

        return load_result

    def get_nb_trainable_parameters(self):
        r"""
        Returns the number of trainable parameters and number of all parameters in the model.
        """
        trainable_params = 0
        all_param = 0
        for param in self.get_parameters():
            num_params = param.numel()
            # if using DS Zero 3 and the weights are initialized empty
            if num_params == 0 and hasattr(param, "ds_numel"):
                num_params = param.ds_numel

            # Due to the design of 4bit linear layers from bitsandbytes
            # one needs to multiply the number of parameters by 2 to get
            # the correct number of parameters
            if param.__class__.__name__ == "Params4bit":
                num_params = num_params * 2

            all_param += num_params
            if param.requires_grad:
                trainable_params += num_params

        return trainable_params, all_param

    def get_prompt_embedding_to_save(self, adapter_name: str) -> mindspore.Tensor:
        """
        Returns the prompt embedding to save when saving the model. Only applicable when using a prompt learning
        method.
        """
        prompt_encoder = self.prompt_encoder[adapter_name]
        prompt_tokens = (
            self.prompt_tokens[adapter_name].unsqueeze(0).expand(1, -1)
        )
        if self.peft_config[adapter_name].peft_type == PeftType.PREFIX_TUNING:
            prompt_tokens = prompt_tokens[:, : self.peft_config[adapter_name].num_virtual_tokens]

        if self.peft_config[adapter_name].peft_type == PeftType.MULTITASK_PROMPT_TUNING:
            prompt_embeddings = super(MultitaskPromptEmbedding, prompt_encoder).forward(prompt_tokens) # pylint: disable=bad-super-call
        else:
            prompt_embeddings = prompt_encoder(prompt_tokens)

        embedding = prompt_embeddings[0]
        return Tensor(embedding.asnumpy())

    def get_prompt(self, batch_size: int, task_ids: Optional[mindspore.Tensor] = None) -> mindspore.Tensor:
        """
        Returns the virtual prompts to use for Peft. Only applicable when using a prompt learning method.
        """
        peft_config = self.active_peft_config
        prompt_encoder = self.prompt_encoder[self.active_adapter]
        prompt_tokens = (
            self.prompt_tokens[self.active_adapter]
            .unsqueeze(0)
            .expand(batch_size, -1)
        )
        if peft_config.peft_type == PeftType.PREFIX_TUNING:
            prompt_tokens = prompt_tokens[:, : peft_config.num_virtual_tokens]
            if peft_config.inference_mode:
                past_key_values = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
            else:
                past_key_values = prompt_encoder(prompt_tokens)
            if self.base_model_dtype is not None:
                past_key_values = past_key_values.to(self.base_model_dtype)
            past_key_values = past_key_values.view(
                batch_size,
                peft_config.num_virtual_tokens,
                peft_config.num_layers * 2,
                peft_config.num_attention_heads,
                peft_config.token_dim // peft_config.num_attention_heads,
            )
            if peft_config.num_transformer_submodules == 2:
                past_key_values = ops.cat([past_key_values, past_key_values], axis=2)
            past_key_values = past_key_values.permute([2, 0, 3, 1, 4]).split(
                peft_config.num_transformer_submodules * 2
            )
            if TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING.get(self.config.model_type, None) is not None:
                post_process_fn = TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING[self.config.model_type]
                past_key_values = post_process_fn(past_key_values)
            return past_key_values
        else:
            if peft_config.peft_type == PeftType.MULTITASK_PROMPT_TUNING:
                prompts = prompt_encoder(prompt_tokens, task_ids)
            else:
                if peft_config.inference_mode:
                    prompts = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
                else:
                    prompts = prompt_encoder(prompt_tokens)
            return prompts

    def print_trainable_parameters(self):
        """
        Prints the number of trainable parameters in the model.
        """
        trainable_params, all_param = self.get_nb_trainable_parameters()

        print(
            f"trainable params: {trainable_params:,d} || all params: {all_param:,d} || trainable%: {100 * trainable_params / all_param}"
        )

    def __getattr__(self, name: str):
        """Forward missing attributes to the wrapped cell."""
        try:
            return super().__getattr__(name)  # defer to nn.Module's logic
        except AttributeError:
            return getattr(self.base_model, name)

    def forward(self, *args, **kwargs):
        """
        Forward pass of the model.
        """
        # print(self.get_base_model().layers[0].__class__.forward)
        return self.get_base_model()(*args, **kwargs)

    def generate(self, *args, **kwargs):
        return self.get_base_model().generate(*args, **kwargs)

    @contextmanager
    def disable_adapter(self):
        """
        Disables the adapter cell.
        """
        try:
            self.base_model.disable_adapter_layers()
            yield
        finally:
            self.base_model.enable_adapter_layers()

    def get_base_model(self):
        """
        Returns the base model.
        """
        return (
            self.base_model
            if self.active_peft_config.is_prompt_learning
            or self.peft_type == PeftType.POLY
            else self.base_model.model
        )

    def add_adapter(self, adapter_name: str, peft_config: PeftConfig):
        """add adapter."""
        if peft_config.peft_type != self.peft_type:
            raise ValueError(
                f"Cannot combine adapters with different peft types. "
                f"Found {self.peft_type} and {peft_config.peft_type}."
            )

        self.peft_config[adapter_name] = peft_config

        try:
            if peft_config.is_prompt_learning:  # add_adapter methods for prompt learning setup
                if hasattr(self.config, "to_dict"):
                    dict_config = self.config.to_dict()
                else:
                    dict_config = self.config

                peft_config = _prepare_prompt_learning_config(peft_config, dict_config)
                self._setup_prompt_encoder(adapter_name)
            # elif peft_config.is_adaption_prompt:
            #     self.base_model.add_adapter(adapter_name, peft_config)
            else:
                # inject adapter into base model (load model instead of initialize new one)
                self.base_model.inject_adapter(self, adapter_name)
        except Exception:  # somthing went wrong, roll back
            del self.peft_config[adapter_name]
            raise

        self.set_additional_trainable_cells(peft_config, adapter_name)

    def set_additional_trainable_cells(self, peft_config, adapter_name):
        """set additional trainable cells"""
        if getattr(peft_config, "cells_to_save", None) is not None:
            if self.cells_to_save is None:
                self.cells_to_save = set(peft_config.cells_to_save)
            else:
                self.cells_to_save.update(peft_config.cells_to_save)
            _set_trainable(self, adapter_name)

    @property
    def active_peft_config(self):
        """active_peft_config"""
        return self.peft_config[self.active_adapter]

mindnlp.peft.peft_model.PeftModel.active_peft_config property

active_peft_config

mindnlp.peft.peft_model.PeftModel.__getattr__(name)

Forward missing attributes to the wrapped cell.

Source code in mindnlp/peft/peft_model.py 
364
365
366
367
368
369
def __getattr__(self, name: str):
    """Forward missing attributes to the wrapped cell."""
    try:
        return super().__getattr__(name)  # defer to nn.Module's logic
    except AttributeError:
        return getattr(self.base_model, name)

mindnlp.peft.peft_model.PeftModel.__init__(model, peft_config, adapter_name='default')

init

This method initializes an instance of the PeftModel class.

PARAMETER DESCRIPTION
self

The instance of the PeftModel class.
model

The base model used for the PeftModel instance.
peft_config

An instance of PeftConfig class containing configuration for the PEFT (Prompt-based Entity Fine-Tuning) process.

TYPE: PeftConfig

adapter_name

The name of the adapter being used. Defaults to 'default'.

TYPE: str DEFAULT: 'default'

RETURNS DESCRIPTION

None. This method does not return any value.
RAISES DESCRIPTION
-TypeError

If the provided model is not of the expected type.
-ValueError

If the provided peft_config is not valid or does not contain necessary information.
-KeyError

If there is an issue with accessing or setting attributes.
Source code in mindnlp/peft/peft_model.py 
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
    r"""
    __init__

    This method initializes an instance of the PeftModel class.

    Args:
        self: The instance of the PeftModel class.
        model: The base model used for the PeftModel instance.
        peft_config (PeftConfig): An instance of PeftConfig class containing configuration for the PEFT (Prompt-based Entity Fine-Tuning) process.
        adapter_name (str, optional): The name of the adapter being used. Defaults to 'default'.

    Returns:
        None. This method does not return any value.

    Raises:
        - TypeError: If the provided model is not of the expected type.
        - ValueError: If the provided peft_config is not valid or does not contain necessary information.
        - KeyError: If there is an issue with accessing or setting attributes.
    """
    super().__init__()
    self.base_model = model
    self.config = getattr(self.base_model, "config", {"model_type": "custom"})
    self.cells_to_save = None
    self.peft_config: Dict[str, PeftConfig] = {}
    self.active_adapter = adapter_name
    self.peft_type = peft_config.peft_type
    self.base_model_dtype = getattr(model, "dtype", None)
    self.special_peft_forward_args = {"adapter_name"}
    if not peft_config.is_prompt_learning:
        self.peft_config[adapter_name] = peft_config
        self.base_model = PEFT_TYPE_TO_MODEL_MAPPING[peft_config.peft_type](
            self.base_model, self.peft_config, adapter_name
        )
        self.set_additional_trainable_cells(peft_config, adapter_name)
    else:
        self.add_adapter(adapter_name, peft_config)

mindnlp.peft.peft_model.PeftModel.add_adapter(adapter_name, peft_config)

add adapter.

Source code in mindnlp/peft/peft_model.py 
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
def add_adapter(self, adapter_name: str, peft_config: PeftConfig):
    """add adapter."""
    if peft_config.peft_type != self.peft_type:
        raise ValueError(
            f"Cannot combine adapters with different peft types. "
            f"Found {self.peft_type} and {peft_config.peft_type}."
        )

    self.peft_config[adapter_name] = peft_config

    try:
        if peft_config.is_prompt_learning:  # add_adapter methods for prompt learning setup
            if hasattr(self.config, "to_dict"):
                dict_config = self.config.to_dict()
            else:
                dict_config = self.config

            peft_config = _prepare_prompt_learning_config(peft_config, dict_config)
            self._setup_prompt_encoder(adapter_name)
        # elif peft_config.is_adaption_prompt:
        #     self.base_model.add_adapter(adapter_name, peft_config)
        else:
            # inject adapter into base model (load model instead of initialize new one)
            self.base_model.inject_adapter(self, adapter_name)
    except Exception:  # somthing went wrong, roll back
        del self.peft_config[adapter_name]
        raise

    self.set_additional_trainable_cells(peft_config, adapter_name)

mindnlp.peft.peft_model.PeftModel.disable_adapter()

Disables the adapter cell.

Source code in mindnlp/peft/peft_model.py 
381
382
383
384
385
386
387
388
389
390
@contextmanager
def disable_adapter(self):
    """
    Disables the adapter cell.
    """
    try:
        self.base_model.disable_adapter_layers()
        yield
    finally:
        self.base_model.enable_adapter_layers()

mindnlp.peft.peft_model.PeftModel.forward(*args, **kwargs)

Forward pass of the model.

Source code in mindnlp/peft/peft_model.py 
371
372
373
374
375
376
def forward(self, *args, **kwargs):
    """
    Forward pass of the model.
    """
    # print(self.get_base_model().layers[0].__class__.forward)
    return self.get_base_model()(*args, **kwargs)

mindnlp.peft.peft_model.PeftModel.from_pretrained(model, model_id, adapter_name='default', is_trainable=False, **kwargs) classmethod

Instantiate a [LoraModel] from a pretrained Lora configuration and weights.

PARAMETER DESCRIPTION
model

The model to be adapted. The model should be initialized with the [~transformers.PreTrainedModel.from_pretrained] method from the 🤗 Transformers library.

TYPE: [`~transformers.PreTrainedModel`]

model_id

The name of the Lora configuration to use. Can be : - A path to a directory containing a Lora configuration file saved using the save_pretrained method (./my_lora_config_directory/).

TYPE: `str` or `os.PathLike`

Source code in mindnlp/peft/peft_model.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
@classmethod
def from_pretrained(cls, model, model_id, adapter_name="default", is_trainable=False, **kwargs):
    r"""
    Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights.

    Args:
        model ([`~transformers.PreTrainedModel`]):
            The model to be adapted. The model should be initialized with the
            [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library.
        model_id (`str` or `os.PathLike`):
            The name of the Lora configuration to use. Can be :
                - A path to a directory containing a Lora configuration file saved using the `save_pretrained`
                  method (`./my_lora_config_directory/`).
    """
    from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING
    # load peft config
    config = PEFT_TYPE_TO_CONFIG_MAPPING[
        PeftConfig.from_pretrained(model_id, subfolder=kwargs.get("subfolder", None)).peft_type
    ].from_pretrained(model_id, subfolder=kwargs.get("subfolder", None))

    config.inference_mode = not is_trainable

    if config.task_type not in MODEL_TYPE_TO_PEFT_MODEL_MAPPING:
        model = cls(model, config, adapter_name)
    else:
        model = MODEL_TYPE_TO_PEFT_MODEL_MAPPING[config.task_type](model, config, adapter_name)
    model.load_adapter(model_id, adapter_name, **kwargs)
    return model

mindnlp.peft.peft_model.PeftModel.get_base_model()

Returns the base model.

Source code in mindnlp/peft/peft_model.py 
392
393
394
395
396
397
398
399
400
401
def get_base_model(self):
    """
    Returns the base model.
    """
    return (
        self.base_model
        if self.active_peft_config.is_prompt_learning
        or self.peft_type == PeftType.POLY
        else self.base_model.model
    )

mindnlp.peft.peft_model.PeftModel.get_nb_trainable_parameters()

Returns the number of trainable parameters and number of all parameters in the model.

Source code in mindnlp/peft/peft_model.py 
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
def get_nb_trainable_parameters(self):
    r"""
    Returns the number of trainable parameters and number of all parameters in the model.
    """
    trainable_params = 0
    all_param = 0
    for param in self.get_parameters():
        num_params = param.numel()
        # if using DS Zero 3 and the weights are initialized empty
        if num_params == 0 and hasattr(param, "ds_numel"):
            num_params = param.ds_numel

        # Due to the design of 4bit linear layers from bitsandbytes
        # one needs to multiply the number of parameters by 2 to get
        # the correct number of parameters
        if param.__class__.__name__ == "Params4bit":
            num_params = num_params * 2

        all_param += num_params
        if param.requires_grad:
            trainable_params += num_params

    return trainable_params, all_param

mindnlp.peft.peft_model.PeftModel.get_prompt(batch_size, task_ids=None)

Returns the virtual prompts to use for Peft. Only applicable when using a prompt learning method.

Source code in mindnlp/peft/peft_model.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
def get_prompt(self, batch_size: int, task_ids: Optional[mindspore.Tensor] = None) -> mindspore.Tensor:
    """
    Returns the virtual prompts to use for Peft. Only applicable when using a prompt learning method.
    """
    peft_config = self.active_peft_config
    prompt_encoder = self.prompt_encoder[self.active_adapter]
    prompt_tokens = (
        self.prompt_tokens[self.active_adapter]
        .unsqueeze(0)
        .expand(batch_size, -1)
    )
    if peft_config.peft_type == PeftType.PREFIX_TUNING:
        prompt_tokens = prompt_tokens[:, : peft_config.num_virtual_tokens]
        if peft_config.inference_mode:
            past_key_values = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
        else:
            past_key_values = prompt_encoder(prompt_tokens)
        if self.base_model_dtype is not None:
            past_key_values = past_key_values.to(self.base_model_dtype)
        past_key_values = past_key_values.view(
            batch_size,
            peft_config.num_virtual_tokens,
            peft_config.num_layers * 2,
            peft_config.num_attention_heads,
            peft_config.token_dim // peft_config.num_attention_heads,
        )
        if peft_config.num_transformer_submodules == 2:
            past_key_values = ops.cat([past_key_values, past_key_values], axis=2)
        past_key_values = past_key_values.permute([2, 0, 3, 1, 4]).split(
            peft_config.num_transformer_submodules * 2
        )
        if TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING.get(self.config.model_type, None) is not None:
            post_process_fn = TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING[self.config.model_type]
            past_key_values = post_process_fn(past_key_values)
        return past_key_values
    else:
        if peft_config.peft_type == PeftType.MULTITASK_PROMPT_TUNING:
            prompts = prompt_encoder(prompt_tokens, task_ids)
        else:
            if peft_config.inference_mode:
                prompts = prompt_encoder.embedding.weight.repeat(batch_size, 1, 1)
            else:
                prompts = prompt_encoder(prompt_tokens)
        return prompts

mindnlp.peft.peft_model.PeftModel.get_prompt_embedding_to_save(adapter_name)

Returns the prompt embedding to save when saving the model. Only applicable when using a prompt learning method.

Source code in mindnlp/peft/peft_model.py 
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
def get_prompt_embedding_to_save(self, adapter_name: str) -> mindspore.Tensor:
    """
    Returns the prompt embedding to save when saving the model. Only applicable when using a prompt learning
    method.
    """
    prompt_encoder = self.prompt_encoder[adapter_name]
    prompt_tokens = (
        self.prompt_tokens[adapter_name].unsqueeze(0).expand(1, -1)
    )
    if self.peft_config[adapter_name].peft_type == PeftType.PREFIX_TUNING:
        prompt_tokens = prompt_tokens[:, : self.peft_config[adapter_name].num_virtual_tokens]

    if self.peft_config[adapter_name].peft_type == PeftType.MULTITASK_PROMPT_TUNING:
        prompt_embeddings = super(MultitaskPromptEmbedding, prompt_encoder).forward(prompt_tokens) # pylint: disable=bad-super-call
    else:
        prompt_embeddings = prompt_encoder(prompt_tokens)

    embedding = prompt_embeddings[0]
    return Tensor(embedding.asnumpy())

mindnlp.peft.peft_model.PeftModel.load_adapter(model_id, adapter_name, is_trainable=False, **kwargs)

load adapter to peft model, called by model.from_pretrained.

Source code in mindnlp/peft/peft_model.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
def load_adapter(self, model_id: str, adapter_name: str, is_trainable: bool = False, **kwargs):
    """load adapter to peft model, called by `model.from_pretrained`."""
    # NOTE: remove download logic.
    if adapter_name not in self.peft_config:
        raise ValueError(f"{adapter_name} is not a valid adapter name. Valid names: {self.peft_config.keys()}")

    adapters_weights = load_peft_weights(model_id)

    # load the weights into the model
    load_result = set_peft_model_state_dict(self, adapters_weights, adapter_name=adapter_name)
    # TODO: add parallel logic & offload logic & device map logic(dispatch_model)

    # Set model in evaluation mode to deactivate Dropout cells by default
    if not is_trainable:
        self.set_train(False)

    return load_result

mindnlp.peft.peft_model.PeftModel.print_trainable_parameters()

Prints the number of trainable parameters in the model.

Source code in mindnlp/peft/peft_model.py 
354
355
356
357
358
359
360
361
362
def print_trainable_parameters(self):
    """
    Prints the number of trainable parameters in the model.
    """
    trainable_params, all_param = self.get_nb_trainable_parameters()

    print(
        f"trainable params: {trainable_params:,d} || all params: {all_param:,d} || trainable%: {100 * trainable_params / all_param}"
    )

mindnlp.peft.peft_model.PeftModel.save_pretrained(save_directory, **kwargs)

This function saves the adapter model and the adapter configuration files to a directory, so that it can be reloaded using the [LoraModel.from_pretrained] class method, and also used by the [LoraModel.push_to_hub] method.

Source code in mindnlp/peft/peft_model.py 
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def save_pretrained(self, save_directory, **kwargs):
    r"""
    This function saves the adapter model and the adapter configuration files to a directory, so that it can be
    reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`]
    method.
    """
    if os.path.isfile(save_directory):
        raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
    os.makedirs(save_directory, exist_ok=True)

    for adapter_name, peft_config in self.peft_config.items():
        # save only the trainable weights
        output_state_dict = get_peft_model_state_dict(
            self,
            state_dict=kwargs.get("state_dict", None),
            adapter_name=adapter_name
        )
        output_dir = os.path.join(save_directory, adapter_name) if adapter_name != "default" else save_directory
        os.makedirs(output_dir, exist_ok=True)

        _exec_save(
            ckpt_file_name=os.path.join(output_dir, WEIGHTS_NAME),
            data_list=output_state_dict,
        )

        # save the config and change the inference mode to `True`
        if peft_config.base_model_name_or_path is None:
            peft_config.base_model_name_or_path = (
                self.base_model.__dict__.get("name_or_path", None),
                self.base_model.model.__dict__.get("name_or_path", None)
            )
        inference_mode = peft_config.inference_mode
        peft_config.inference_mode = True
        peft_config.save_pretrained(output_dir)
        peft_config.inference_mode = inference_mode

mindnlp.peft.peft_model.PeftModel.set_additional_trainable_cells(peft_config, adapter_name)

set additional trainable cells

Source code in mindnlp/peft/peft_model.py 
433
434
435
436
437
438
439
440
def set_additional_trainable_cells(self, peft_config, adapter_name):
    """set additional trainable cells"""
    if getattr(peft_config, "cells_to_save", None) is not None:
        if self.cells_to_save is None:
            self.cells_to_save = set(peft_config.cells_to_save)
        else:
            self.cells_to_save.update(peft_config.cells_to_save)
        _set_trainable(self, adapter_name)

mindnlp.peft.peft_model.PeftModelForCausalLM

Bases: PeftModel

Peft model for causal language modeling.

PARAMETER DESCRIPTION
model

Base transformer model.

TYPE: [`~mindnlp.models.PreTrainedModel`]

peft_config

Peft config.

TYPE: [`PeftConfig`]

Source code in mindnlp/peft/peft_model.py 
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
class PeftModelForCausalLM(PeftModel):
    """
    Peft model for causal language modeling.

    Args:
        model ([`~mindnlp.models.PreTrainedModel`]): Base transformer model.
        peft_config ([`PeftConfig`]): Peft config.
    """
    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
        r"""
        Initializes a new instance of the PeftModelForCausalLM class.

        Args:
            self: The instance itself.
            model: The underlying model for the adapter.
            peft_config (PeftConfig): The configuration for the PEFT (Plug and Fine-tune) adapter.
            adapter_name (str): The name of the adapter. Defaults to 'default'.

        Returns:
            None. This method does not return any value.

        Raises:
            N/A
        """
        super().__init__(model, peft_config, adapter_name)
        self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
        task_ids=None,
        **kwargs,
    ):
        """
        Forward pass of the model.
        """
        peft_config = self.active_peft_config
        if not isinstance(peft_config, PromptLearningConfig):
            if self.base_model.config.model_type == "mpt":
                if inputs_embeds is not None:
                    raise AssertionError("forward in MPTForCausalLM does not support inputs_embeds")
                return self.base_model(
                    input_ids=input_ids,
                    attention_mask=attention_mask,
                    labels=labels,
                    output_attentions=output_attentions,
                    output_hidden_states=output_hidden_states,
                    return_dict=return_dict,
                    **kwargs,
                )
            if peft_config.peft_type == PeftType.POLY:
                kwargs["task_ids"] = task_ids
            return self.base_model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                labels=labels,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                **kwargs,
            )

        batch_size = input_ids.shape[0]
        if attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
            attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)

        if kwargs.get("position_ids", None) is not None:
            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
            kwargs["position_ids"] = None
        if kwargs.get("token_type_ids", None) is not None:
            warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
            kwargs["token_type_ids"] = None
        kwargs.update(
            {
                "attention_mask": attention_mask,
                "labels": labels,
                "output_attentions": output_attentions,
                "output_hidden_states": output_hidden_states,
                "return_dict": return_dict,
            }
        )

        if peft_config.peft_type == PeftType.PREFIX_TUNING:
            past_key_values = self.get_prompt(batch_size)
            return self.base_model(input_ids=input_ids, past_key_values=past_key_values, **kwargs)
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        # concat prompt labels
        if labels is not None:
            prefix_labels = ops.full((batch_size, peft_config.num_virtual_tokens), -100)
            kwargs["labels"] = ops.cat((prefix_labels, labels), axis=1)
        prompts = self.get_prompt(batch_size=batch_size)
        prompts = prompts.to(inputs_embeds.dtype)
        inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
        return self.base_model(inputs_embeds=inputs_embeds, **kwargs)

    def generate(self, **kwargs):
        """generate."""
        self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
        if hasattr(self.base_model, "model"):
            self.base_model.model.generation_config = self.generation_config
        else:
            self.base_model.generation_config = self.generation_config
        try:
            outputs = self.base_model.generate(**kwargs)
        except:
            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
            raise

        self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
        return outputs

    def prepare_inputs_for_generation(self, *args, task_ids: Optional[mindspore.Tensor] = None, **kwargs,):
        """prepare_inputs_for_generation."""
        peft_config = self.active_peft_config
        model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
        if peft_config.peft_type == PeftType.POLY:
            model_kwargs["task_ids"] = task_ids
        if isinstance(peft_config, PromptLearningConfig):
            if model_kwargs.get("attention_mask", None) is not None:
                prefix_attention_mask = ops.ones(
                    model_kwargs["input_ids"].shape[0], peft_config.num_virtual_tokens)
                model_kwargs["attention_mask"] = ops.cat(
                    (prefix_attention_mask, model_kwargs["attention_mask"]), axis=1
                )

            if model_kwargs.get("position_ids", None) is not None:
                warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
                model_kwargs["position_ids"] = None

            if kwargs.get("token_type_ids", None) is not None:
                warnings.warn(
                    "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
                )
                kwargs["token_type_ids"] = None

            if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
                past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
                model_kwargs["past_key_values"] = past_key_values
            else:
                if model_kwargs["past_key_values"] is None:
                    inputs_embeds = self.word_embeddings(model_kwargs["input_ids"])
                    prompts = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
                    prompts = prompts.to(inputs_embeds.dtype)
                    model_kwargs["inputs_embeds"] = ops.cat((prompts, inputs_embeds), axis=1)
                    model_kwargs["input_ids"] = None

        return model_kwargs

mindnlp.peft.peft_model.PeftModelForCausalLM.__init__(model, peft_config, adapter_name='default')

Initializes a new instance of the PeftModelForCausalLM class.

PARAMETER DESCRIPTION
self

The instance itself.
model

The underlying model for the adapter.
peft_config

The configuration for the PEFT (Plug and Fine-tune) adapter.

TYPE: PeftConfig

adapter_name

The name of the adapter. Defaults to 'default'.

TYPE: str DEFAULT: 'default'

RETURNS DESCRIPTION

None. This method does not return any value.
Source code in mindnlp/peft/peft_model.py 
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
    r"""
    Initializes a new instance of the PeftModelForCausalLM class.

    Args:
        self: The instance itself.
        model: The underlying model for the adapter.
        peft_config (PeftConfig): The configuration for the PEFT (Plug and Fine-tune) adapter.
        adapter_name (str): The name of the adapter. Defaults to 'default'.

    Returns:
        None. This method does not return any value.

    Raises:
        N/A
    """
    super().__init__(model, peft_config, adapter_name)
    self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation

mindnlp.peft.peft_model.PeftModelForCausalLM.forward(input_ids=None, attention_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None, task_ids=None, **kwargs)

Forward pass of the model.

Source code in mindnlp/peft/peft_model.py 
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
def forward(
    self,
    input_ids=None,
    attention_mask=None,
    inputs_embeds=None,
    labels=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    task_ids=None,
    **kwargs,
):
    """
    Forward pass of the model.
    """
    peft_config = self.active_peft_config
    if not isinstance(peft_config, PromptLearningConfig):
        if self.base_model.config.model_type == "mpt":
            if inputs_embeds is not None:
                raise AssertionError("forward in MPTForCausalLM does not support inputs_embeds")
            return self.base_model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=labels,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                **kwargs,
            )
        if peft_config.peft_type == PeftType.POLY:
            kwargs["task_ids"] = task_ids
        return self.base_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            labels=labels,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            **kwargs,
        )

    batch_size = input_ids.shape[0]
    if attention_mask is not None:
        # concat prompt attention mask
        prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
        attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)

    if kwargs.get("position_ids", None) is not None:
        warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
        kwargs["position_ids"] = None
    if kwargs.get("token_type_ids", None) is not None:
        warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
        kwargs["token_type_ids"] = None
    kwargs.update(
        {
            "attention_mask": attention_mask,
            "labels": labels,
            "output_attentions": output_attentions,
            "output_hidden_states": output_hidden_states,
            "return_dict": return_dict,
        }
    )

    if peft_config.peft_type == PeftType.PREFIX_TUNING:
        past_key_values = self.get_prompt(batch_size)
        return self.base_model(input_ids=input_ids, past_key_values=past_key_values, **kwargs)
    if inputs_embeds is None:
        inputs_embeds = self.word_embeddings(input_ids)
    # concat prompt labels
    if labels is not None:
        prefix_labels = ops.full((batch_size, peft_config.num_virtual_tokens), -100)
        kwargs["labels"] = ops.cat((prefix_labels, labels), axis=1)
    prompts = self.get_prompt(batch_size=batch_size)
    prompts = prompts.to(inputs_embeds.dtype)
    inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
    return self.base_model(inputs_embeds=inputs_embeds, **kwargs)

mindnlp.peft.peft_model.PeftModelForCausalLM.generate(**kwargs)

generate.

Source code in mindnlp/peft/peft_model.py 
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
def generate(self, **kwargs):
    """generate."""
    self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
    if hasattr(self.base_model, "model"):
        self.base_model.model.generation_config = self.generation_config
    else:
        self.base_model.generation_config = self.generation_config
    try:
        outputs = self.base_model.generate(**kwargs)
    except:
        self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
        raise

    self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
    return outputs

mindnlp.peft.peft_model.PeftModelForCausalLM.prepare_inputs_for_generation(*args, task_ids=None, **kwargs)

prepare_inputs_for_generation.

Source code in mindnlp/peft/peft_model.py 
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
def prepare_inputs_for_generation(self, *args, task_ids: Optional[mindspore.Tensor] = None, **kwargs,):
    """prepare_inputs_for_generation."""
    peft_config = self.active_peft_config
    model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
    if peft_config.peft_type == PeftType.POLY:
        model_kwargs["task_ids"] = task_ids
    if isinstance(peft_config, PromptLearningConfig):
        if model_kwargs.get("attention_mask", None) is not None:
            prefix_attention_mask = ops.ones(
                model_kwargs["input_ids"].shape[0], peft_config.num_virtual_tokens)
            model_kwargs["attention_mask"] = ops.cat(
                (prefix_attention_mask, model_kwargs["attention_mask"]), axis=1
            )

        if model_kwargs.get("position_ids", None) is not None:
            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
            model_kwargs["position_ids"] = None

        if kwargs.get("token_type_ids", None) is not None:
            warnings.warn(
                "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
            )
            kwargs["token_type_ids"] = None

        if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
            past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
            model_kwargs["past_key_values"] = past_key_values
        else:
            if model_kwargs["past_key_values"] is None:
                inputs_embeds = self.word_embeddings(model_kwargs["input_ids"])
                prompts = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0])
                prompts = prompts.to(inputs_embeds.dtype)
                model_kwargs["inputs_embeds"] = ops.cat((prompts, inputs_embeds), axis=1)
                model_kwargs["input_ids"] = None

    return model_kwargs

mindnlp.peft.peft_model.PeftModelForSeq2SeqLM

Bases: PeftModel

Peft model for sequence-to-sequence language modeling.

PARAMETER DESCRIPTION
model

Base transformer model.

TYPE: [`~transformers.PreTrainedModel`]

peft_config

Peft config.

TYPE: [`PeftConfig`]

Source code in mindnlp/peft/peft_model.py 
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
class PeftModelForSeq2SeqLM(PeftModel):
    """
    Peft model for sequence-to-sequence language modeling.

    Args:
        model ([`~transformers.PreTrainedModel`]): Base transformer model.
        peft_config ([`PeftConfig`]): Peft config.

    """
    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
        r"""
        Initialize a new PeftModelForSeq2SeqLM object.

        Args:
            self: The instance of the PeftModelForSeq2SeqLM class.
            model: The model to be used for the PeftModelForSeq2SeqLM.
            peft_config (PeftConfig): The configuration object for the PeftModelForSeq2SeqLM.
            adapter_name (str): The name of the adapter to be used, defaults to 'default'.

        Returns:
            None. This method initializes the PeftModelForSeq2SeqLM object.

        Raises:
            None.
        """
        super().__init__(model, peft_config, adapter_name)
        self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
        self.base_model_prepare_encoder_decoder_kwargs_for_generation = (
            self.base_model._prepare_encoder_decoder_kwargs_for_generation
        )

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        decoder_input_ids=None,
        decoder_attention_mask=None,
        decoder_inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
        task_ids=None,
        **kwargs,
    ):
        """
        Forward pass of the model.
        """
        peft_config = self.active_peft_config
        if not isinstance(peft_config, PromptLearningConfig):
            return self.base_model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                decoder_input_ids=decoder_input_ids,
                decoder_attention_mask=decoder_attention_mask,
                decoder_inputs_embeds=decoder_inputs_embeds,
                labels=labels,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                **kwargs,
            )

        batch_size = input_ids.shape[0]
        if decoder_attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
            decoder_attention_mask = ops.cat((prefix_attention_mask, decoder_attention_mask), axis=1)

        if kwargs.get("position_ids", None) is not None:
            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
            kwargs["position_ids"] = None
        if kwargs.get("token_type_ids", None) is not None:
            warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
            kwargs["token_type_ids"] = None
        kwargs.update(
            {
                "attention_mask": attention_mask,
                "decoder_attention_mask": decoder_attention_mask,
                "labels": labels,
                "output_attentions": output_attentions,
                "output_hidden_states": output_hidden_states,
                "return_dict": return_dict,
            }
        )

        if peft_config.peft_type == PeftType.PREFIX_TUNING:
            past_key_values = self.get_prompt(batch_size)
            return self.base_model(
                input_ids=input_ids, decoder_input_ids=decoder_input_ids, past_key_values=past_key_values, **kwargs
            )
        elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
            if inputs_embeds is None:
                inputs_embeds = self.word_embeddings(input_ids)

            if attention_mask is not None:
                # concat prompt attention mask
                prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
                kwargs["attention_mask"] = ops.cat((prefix_attention_mask, attention_mask), axis=1)

            prompts = self.get_prompt(batch_size=batch_size)
            prompts = prompts.to(inputs_embeds.dtype)
            inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)

            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
        else:
            if inputs_embeds is None:
                inputs_embeds = self.word_embeddings(input_ids)
            if decoder_inputs_embeds is None and decoder_input_ids is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )
                decoder_inputs_embeds = self.word_embeddings(decoder_input_ids)

            if attention_mask is not None:
                # concat prompt attention mask
                prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens, dtype=attention_mask.dtype)
                kwargs["attention_mask"] = ops.cat((prefix_attention_mask, attention_mask), axis=1)
            # concat prompt labels
            if labels is not None:
                if peft_config.num_transformer_submodules == 1:
                    kwargs["labels"] = labels
                elif peft_config.num_transformer_submodules == 2:
                    prefix_labels = ops.full((batch_size, peft_config.num_virtual_tokens), -100)
                    kwargs["labels"] = ops.cat((prefix_labels, labels), axis=1)
            prompts = self.get_prompt(batch_size=batch_size, task_ids=task_ids)
            prompts = prompts.to(inputs_embeds.dtype)
            inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)
            if peft_config.num_transformer_submodules == 1:
                return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
            elif peft_config.num_transformer_submodules == 2:
                decoder_inputs_embeds = ops.cat(
                    (prompts[:, peft_config.num_virtual_tokens :], decoder_inputs_embeds), axis=1
                )
                return self.base_model(
                    inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs
                )
            return None # never go here

    def generate(self, **kwargs):
        """generate."""
        peft_config = self.active_peft_config
        self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
        self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
            self._prepare_encoder_decoder_kwargs_for_generation
        )
        try:
            if not isinstance(peft_config, PromptLearningConfig):
                outputs = self.base_model.generate(**kwargs)
            else:
                if "input_ids" not in kwargs:
                    raise ValueError("input_ids must be provided for Peft model generation")
                if kwargs.get("position_ids", None) is not None:
                    warnings.warn(
                        "Position ids are not supported for parameter efficient tuning. Ignoring position ids."
                    )
                    kwargs["position_ids"] = None
                if kwargs.get("token_type_ids", None) is not None:
                    warnings.warn(
                        "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
                    )
                    kwargs["token_type_ids"] = None

                if peft_config.peft_type == PeftType.PREFIX_TUNING:
                    outputs = self.base_model.generate(**kwargs)
                elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
                    kwargs = deepcopy(kwargs)

                    if "encoder_outputs" in kwargs:
                        del kwargs["encoder_ouputs"]
                        warnings.warn(
                            "`encoder_outputs` should not be passed to `generate` when using prompt tuning. Ignoring it."
                        )

                    input_ids = kwargs.pop("input_ids")
                    inputs_embeds = self.word_embeddings(input_ids)
                    batch_size = inputs_embeds.shape[0]
                    prompts = self.get_prompt(batch_size=batch_size)
                    prompts = prompts.to(inputs_embeds.dtype)

                    inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)
                    kwargs["inputs_embeds"] = inputs_embeds

                    if "attention_mask" in kwargs:
                        prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
                        kwargs["attention_mask"] = ops.cat((prefix_attention_mask, kwargs["attention_mask"]), axis=1)

                    return self.base_model.generate(**kwargs)
                else:
                    raise NotImplementedError
        except:
            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
            self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
                self.base_model_prepare_encoder_decoder_kwargs_for_generation
            )
            raise
        self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
        self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
            self.base_model_prepare_encoder_decoder_kwargs_for_generation
        )
        return outputs

    def prepare_inputs_for_generation(self, *args, task_ids: mindspore.Tensor = None, **kwargs):
        """prepare inputs for generation"""
        peft_config = self.active_peft_config
        model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
        if peft_config.peft_type == PeftType.POLY:
            model_kwargs["task_ids"] = task_ids
        if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
            batch_size = model_kwargs["decoder_input_ids"].shape[0]
            past_key_values = self.get_prompt(batch_size)
            model_kwargs["past_key_values"] = past_key_values

        return model_kwargs

mindnlp.peft.peft_model.PeftModelForSeq2SeqLM.__init__(model, peft_config, adapter_name='default')

Initialize a new PeftModelForSeq2SeqLM object.

PARAMETER DESCRIPTION
self

The instance of the PeftModelForSeq2SeqLM class.
model

The model to be used for the PeftModelForSeq2SeqLM.
peft_config

The configuration object for the PeftModelForSeq2SeqLM.

TYPE: PeftConfig

adapter_name

The name of the adapter to be used, defaults to 'default'.

TYPE: str DEFAULT: 'default'

RETURNS DESCRIPTION

None. This method initializes the PeftModelForSeq2SeqLM object.
Source code in mindnlp/peft/peft_model.py 
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
    r"""
    Initialize a new PeftModelForSeq2SeqLM object.

    Args:
        self: The instance of the PeftModelForSeq2SeqLM class.
        model: The model to be used for the PeftModelForSeq2SeqLM.
        peft_config (PeftConfig): The configuration object for the PeftModelForSeq2SeqLM.
        adapter_name (str): The name of the adapter to be used, defaults to 'default'.

    Returns:
        None. This method initializes the PeftModelForSeq2SeqLM object.

    Raises:
        None.
    """
    super().__init__(model, peft_config, adapter_name)
    self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
    self.base_model_prepare_encoder_decoder_kwargs_for_generation = (
        self.base_model._prepare_encoder_decoder_kwargs_for_generation
    )

mindnlp.peft.peft_model.PeftModelForSeq2SeqLM.forward(input_ids=None, attention_mask=None, inputs_embeds=None, decoder_input_ids=None, decoder_attention_mask=None, decoder_inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None, task_ids=None, **kwargs)

Forward pass of the model.

Source code in mindnlp/peft/peft_model.py 
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
def forward(
    self,
    input_ids=None,
    attention_mask=None,
    inputs_embeds=None,
    decoder_input_ids=None,
    decoder_attention_mask=None,
    decoder_inputs_embeds=None,
    labels=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    task_ids=None,
    **kwargs,
):
    """
    Forward pass of the model.
    """
    peft_config = self.active_peft_config
    if not isinstance(peft_config, PromptLearningConfig):
        return self.base_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            decoder_inputs_embeds=decoder_inputs_embeds,
            labels=labels,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            **kwargs,
        )

    batch_size = input_ids.shape[0]
    if decoder_attention_mask is not None:
        # concat prompt attention mask
        prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
        decoder_attention_mask = ops.cat((prefix_attention_mask, decoder_attention_mask), axis=1)

    if kwargs.get("position_ids", None) is not None:
        warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
        kwargs["position_ids"] = None
    if kwargs.get("token_type_ids", None) is not None:
        warnings.warn("Token type ids are not supported for parameter efficient tuning. Ignoring token type ids")
        kwargs["token_type_ids"] = None
    kwargs.update(
        {
            "attention_mask": attention_mask,
            "decoder_attention_mask": decoder_attention_mask,
            "labels": labels,
            "output_attentions": output_attentions,
            "output_hidden_states": output_hidden_states,
            "return_dict": return_dict,
        }
    )

    if peft_config.peft_type == PeftType.PREFIX_TUNING:
        past_key_values = self.get_prompt(batch_size)
        return self.base_model(
            input_ids=input_ids, decoder_input_ids=decoder_input_ids, past_key_values=past_key_values, **kwargs
        )
    elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)

        if attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
            kwargs["attention_mask"] = ops.cat((prefix_attention_mask, attention_mask), axis=1)

        prompts = self.get_prompt(batch_size=batch_size)
        prompts = prompts.to(inputs_embeds.dtype)
        inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)

        return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
    else:
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        if decoder_inputs_embeds is None and decoder_input_ids is None:
            decoder_input_ids = shift_tokens_right(
                labels, self.config.pad_token_id, self.config.decoder_start_token_id
            )
            decoder_inputs_embeds = self.word_embeddings(decoder_input_ids)

        if attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens, dtype=attention_mask.dtype)
            kwargs["attention_mask"] = ops.cat((prefix_attention_mask, attention_mask), axis=1)
        # concat prompt labels
        if labels is not None:
            if peft_config.num_transformer_submodules == 1:
                kwargs["labels"] = labels
            elif peft_config.num_transformer_submodules == 2:
                prefix_labels = ops.full((batch_size, peft_config.num_virtual_tokens), -100)
                kwargs["labels"] = ops.cat((prefix_labels, labels), axis=1)
        prompts = self.get_prompt(batch_size=batch_size, task_ids=task_ids)
        prompts = prompts.to(inputs_embeds.dtype)
        inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)
        if peft_config.num_transformer_submodules == 1:
            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)
        elif peft_config.num_transformer_submodules == 2:
            decoder_inputs_embeds = ops.cat(
                (prompts[:, peft_config.num_virtual_tokens :], decoder_inputs_embeds), axis=1
            )
            return self.base_model(
                inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs
            )
        return None # never go here

mindnlp.peft.peft_model.PeftModelForSeq2SeqLM.generate(**kwargs)

generate.

Source code in mindnlp/peft/peft_model.py 
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
def generate(self, **kwargs):
    """generate."""
    peft_config = self.active_peft_config
    self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
    self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
        self._prepare_encoder_decoder_kwargs_for_generation
    )
    try:
        if not isinstance(peft_config, PromptLearningConfig):
            outputs = self.base_model.generate(**kwargs)
        else:
            if "input_ids" not in kwargs:
                raise ValueError("input_ids must be provided for Peft model generation")
            if kwargs.get("position_ids", None) is not None:
                warnings.warn(
                    "Position ids are not supported for parameter efficient tuning. Ignoring position ids."
                )
                kwargs["position_ids"] = None
            if kwargs.get("token_type_ids", None) is not None:
                warnings.warn(
                    "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
                )
                kwargs["token_type_ids"] = None

            if peft_config.peft_type == PeftType.PREFIX_TUNING:
                outputs = self.base_model.generate(**kwargs)
            elif peft_config.peft_type in [PeftType.PROMPT_TUNING, PeftType.P_TUNING]:
                kwargs = deepcopy(kwargs)

                if "encoder_outputs" in kwargs:
                    del kwargs["encoder_ouputs"]
                    warnings.warn(
                        "`encoder_outputs` should not be passed to `generate` when using prompt tuning. Ignoring it."
                    )

                input_ids = kwargs.pop("input_ids")
                inputs_embeds = self.word_embeddings(input_ids)
                batch_size = inputs_embeds.shape[0]
                prompts = self.get_prompt(batch_size=batch_size)
                prompts = prompts.to(inputs_embeds.dtype)

                inputs_embeds = ops.cat((prompts[:, : peft_config.num_virtual_tokens], inputs_embeds), axis=1)
                kwargs["inputs_embeds"] = inputs_embeds

                if "attention_mask" in kwargs:
                    prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
                    kwargs["attention_mask"] = ops.cat((prefix_attention_mask, kwargs["attention_mask"]), axis=1)

                return self.base_model.generate(**kwargs)
            else:
                raise NotImplementedError
    except:
        self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
        self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
            self.base_model_prepare_encoder_decoder_kwargs_for_generation
        )
        raise
    self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
    self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
        self.base_model_prepare_encoder_decoder_kwargs_for_generation
    )
    return outputs

mindnlp.peft.peft_model.PeftModelForSeq2SeqLM.prepare_inputs_for_generation(*args, task_ids=None, **kwargs)

prepare inputs for generation

Source code in mindnlp/peft/peft_model.py 
918
919
920
921
922
923
924
925
926
927
928
929
def prepare_inputs_for_generation(self, *args, task_ids: mindspore.Tensor = None, **kwargs):
    """prepare inputs for generation"""
    peft_config = self.active_peft_config
    model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
    if peft_config.peft_type == PeftType.POLY:
        model_kwargs["task_ids"] = task_ids
    if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
        batch_size = model_kwargs["decoder_input_ids"].shape[0]
        past_key_values = self.get_prompt(batch_size)
        model_kwargs["past_key_values"] = past_key_values

    return model_kwargs

mindnlp.peft.peft_model.PeftModelForSequenceClassification

Bases: PeftModel

Peft model for sequence classification tasks.

PARAMETER DESCRIPTION
model

Base transformer model.

TYPE: [`~mindnlp.models.PreTrainedModel`]

peft_config

Peft config.

TYPE: [`PeftConfig`]

Source code in mindnlp/peft/peft_model.py 
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
class PeftModelForSequenceClassification(PeftModel):
    """
    Peft model for sequence classification tasks.

    Args:
        model ([`~mindnlp.models.PreTrainedModel`]): Base transformer model.
        peft_config ([`PeftConfig`]): Peft config.

    """
    def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
        """
        Initializes a new instance of the PeftModelForSequenceClassification class.

        Args:
            self: The instance of the PeftModelForSequenceClassification class.
            model: The base model to be used for sequence classification (e.g., a pre-trained language model).
            peft_config (PeftConfig): The configuration for the PEFT (Probing and Evaluation for Transformers) model.
            adapter_name (str, optional): The name of the adapter to be used. Defaults to 'default'.

        Returns:
            None. This method initializes the instance with the specified parameters.

        Raises:
            None.
        """
        super().__init__(model, peft_config, adapter_name)
        if self.cells_to_save is None:
            self.cells_to_save = {"classifier", "score"}
        else:
            self.cells_to_save.update({"classifier", "score"})

        for name, _ in self.base_model.cells_and_names():
            if any(cell_name in name for cell_name in self.cells_to_save):
                self.cls_layer_name = name
                break

        # to make sure classifier layer is trainable
        _set_trainable(self, adapter_name)

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
        task_ids=None,
        **kwargs,
    ):
        """
        Forward pass of the model.
        """
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
        peft_config = self.active_peft_config
        if not peft_config.is_prompt_learning:
            # NOTE:some args not exists in base model
            # inputs_embeds=inputs_embeds,
            # output_attentions=output_attentions,
            # output_hidden_states=output_hidden_states,
            # return_dict=return_dict,

            if peft_config.peft_type == PeftType.POLY:
                kwargs["task_ids"] = task_ids
            return self.base_model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=labels,
                **kwargs,
            )

        batch_size = _get_batch_size(input_ids, inputs_embeds)
        if attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens, dtype=attention_mask.dtype)
            attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)
        if kwargs.get("position_ids", None) is not None:
            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
            kwargs["position_ids"] = None
        kwargs.update(
            {
                "attention_mask": attention_mask,
                "labels": labels,
                "output_attentions": output_attentions,
                "output_hidden_states": output_hidden_states,
                "return_dict": return_dict,
            }
        )

        # if peft_config.peft_type == PeftType.PREFIX_TUNING:
        #     return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
        if kwargs.get("token_type_ids", None) is not None:
            kwargs["token_type_ids"] = ops.cat(
                (
                    ops.zeros(batch_size, peft_config.num_virtual_tokens, dtype=kwargs["token_type_ids"].dtype),
                    kwargs["token_type_ids"],
                ),
                axis=1,
            )
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        prompts = self.get_prompt(batch_size=batch_size)
        prompts = prompts.to(inputs_embeds.dtype)
        inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
        return self.base_model(inputs_embeds=inputs_embeds, **kwargs)

mindnlp.peft.peft_model.PeftModelForSequenceClassification.__init__(model, peft_config, adapter_name='default')

Initializes a new instance of the PeftModelForSequenceClassification class.

PARAMETER DESCRIPTION
self

The instance of the PeftModelForSequenceClassification class.
model

The base model to be used for sequence classification (e.g., a pre-trained language model).
peft_config

The configuration for the PEFT (Probing and Evaluation for Transformers) model.

TYPE: PeftConfig

adapter_name

The name of the adapter to be used. Defaults to 'default'.

TYPE: str DEFAULT: 'default'

RETURNS DESCRIPTION

None. This method initializes the instance with the specified parameters.
Source code in mindnlp/peft/peft_model.py 
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
    """
    Initializes a new instance of the PeftModelForSequenceClassification class.

    Args:
        self: The instance of the PeftModelForSequenceClassification class.
        model: The base model to be used for sequence classification (e.g., a pre-trained language model).
        peft_config (PeftConfig): The configuration for the PEFT (Probing and Evaluation for Transformers) model.
        adapter_name (str, optional): The name of the adapter to be used. Defaults to 'default'.

    Returns:
        None. This method initializes the instance with the specified parameters.

    Raises:
        None.
    """
    super().__init__(model, peft_config, adapter_name)
    if self.cells_to_save is None:
        self.cells_to_save = {"classifier", "score"}
    else:
        self.cells_to_save.update({"classifier", "score"})

    for name, _ in self.base_model.cells_and_names():
        if any(cell_name in name for cell_name in self.cells_to_save):
            self.cls_layer_name = name
            break

    # to make sure classifier layer is trainable
    _set_trainable(self, adapter_name)

mindnlp.peft.peft_model.PeftModelForSequenceClassification.forward(input_ids=None, attention_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None, task_ids=None, **kwargs)

Forward pass of the model.

Source code in mindnlp/peft/peft_model.py 
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
def forward(
    self,
    input_ids=None,
    attention_mask=None,
    inputs_embeds=None,
    labels=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    task_ids=None,
    **kwargs,
):
    """
    Forward pass of the model.
    """
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
    peft_config = self.active_peft_config
    if not peft_config.is_prompt_learning:
        # NOTE:some args not exists in base model
        # inputs_embeds=inputs_embeds,
        # output_attentions=output_attentions,
        # output_hidden_states=output_hidden_states,
        # return_dict=return_dict,

        if peft_config.peft_type == PeftType.POLY:
            kwargs["task_ids"] = task_ids
        return self.base_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            labels=labels,
            **kwargs,
        )

    batch_size = _get_batch_size(input_ids, inputs_embeds)
    if attention_mask is not None:
        # concat prompt attention mask
        prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens, dtype=attention_mask.dtype)
        attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)
    if kwargs.get("position_ids", None) is not None:
        warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
        kwargs["position_ids"] = None
    kwargs.update(
        {
            "attention_mask": attention_mask,
            "labels": labels,
            "output_attentions": output_attentions,
            "output_hidden_states": output_hidden_states,
            "return_dict": return_dict,
        }
    )

    # if peft_config.peft_type == PeftType.PREFIX_TUNING:
    #     return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
    if kwargs.get("token_type_ids", None) is not None:
        kwargs["token_type_ids"] = ops.cat(
            (
                ops.zeros(batch_size, peft_config.num_virtual_tokens, dtype=kwargs["token_type_ids"].dtype),
                kwargs["token_type_ids"],
            ),
            axis=1,
        )
    if inputs_embeds is None:
        inputs_embeds = self.word_embeddings(input_ids)
    prompts = self.get_prompt(batch_size=batch_size)
    prompts = prompts.to(inputs_embeds.dtype)
    inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
    return self.base_model(inputs_embeds=inputs_embeds, **kwargs)

mindnlp.peft.peft_model.PeftModelForTokenClassification

Bases: PeftModel

Peft model for token classification tasks.

PARAMETER DESCRIPTION
model

Base transformer model.

TYPE: [`~transformers.PreTrainedModel`]

peft_config

Peft config.

TYPE: [`PeftConfig`] DEFAULT: None

Source code in mindnlp/peft/peft_model.py 
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
class PeftModelForTokenClassification(PeftModel):
    """
    Peft model for token classification tasks.

    Args:
        model ([`~transformers.PreTrainedModel`]): Base transformer model.
        peft_config ([`PeftConfig`]): Peft config.
    """
    def __init__(self, model, peft_config: PeftConfig = None, adapter_name="default"):
        r"""
        Initializes a new instance of the PeftModelForTokenClassification class.

        Args:
            self: The instance of the PeftModelForTokenClassification class.
            model: The model used for token classification.
            peft_config (PeftConfig, optional): The configuration for the Peft model. Defaults to None.
            adapter_name (str, optional): The name of the adapter. Defaults to 'default'.

        Returns:
            None. This method does not return a value.

        Raises:
            N/A
        """
        super().__init__(model, peft_config, adapter_name)
        if self.cells_to_save is None:
            self.cells_to_save = {"classifier", "score"}
        else:
            self.cells_to_save.update({"classifier", "score"})

        for name, _ in self.base_model.cells_and_names():
            if any(cell_name in name for cell_name in self.cells_to_save):
                self.cls_layer_name = name
                break

        # to make sure classifier layer is trainable
        _set_trainable(self, adapter_name)

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
        task_ids=None,
        **kwargs,
    ):
        """
        Forward pass of the model.
        """
        peft_config = self.active_peft_config
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if not isinstance(peft_config, PromptLearningConfig):
            if peft_config.peft_type == PeftType.POLY:
                kwargs["task_ids"] = task_ids
            return self.base_model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                labels=labels,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                **kwargs,
            )

        batch_size = input_ids.shape[0]
        if attention_mask is not None:
            # concat prompt attention mask
            prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
            attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)
        if kwargs.get("position_ids", None) is not None:
            warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
            kwargs["position_ids"] = None
        kwargs.update(
            {
                "attention_mask": attention_mask,
                "labels": labels,
                "output_attentions": output_attentions,
                "output_hidden_states": output_hidden_states,
                "return_dict": return_dict,
            }
        )

        if peft_config.peft_type == PeftType.PREFIX_TUNING:
            return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
        else:
            if kwargs.get("token_type_ids", None) is not None:
                kwargs["token_type_ids"] = ops.cat(
                    (
                        ops.zeros(batch_size, peft_config.num_virtual_tokens),
                        kwargs["token_type_ids"],
                    ),
                    axis=1,
                ).long()
            if inputs_embeds is None:
                inputs_embeds = self.word_embeddings(input_ids)
            prompts = self.get_prompt(batch_size=batch_size)
            prompts = prompts.to(inputs_embeds.dtype)
            inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
            return self.base_model(inputs_embeds=inputs_embeds, **kwargs)

    def _prefix_tuning_forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
        **kwargs,
    ):
        r"""
        Performs the forward pass for the prefix tuning task in the PeftModelForTokenClassification class.

        Args:
            self (PeftModelForTokenClassification): The instance of the PeftModelForTokenClassification class.
            input_ids (torch.Tensor): The input token IDs tensor of shape [batch_size, sequence_length].
            attention_mask (torch.Tensor): The attention mask tensor of shape [batch_size, sequence_length].
            inputs_embeds (torch.Tensor): The input embeddings tensor of shape [batch_size, sequence_length, hidden_size].
            labels (torch.Tensor): The labels tensor of shape [batch_size, sequence_length].
            output_attentions (bool): Whether to output attentions. Defaults to None.
            output_hidden_states (bool): Whether to output hidden states. Defaults to None.
            return_dict (bool): Whether to return a dictionary. Defaults to None.

        Returns:
            None: This method does not return any value. Instead, it updates the internal state of the model.

        Raises:
            ValueError: If the model does not support past key values which are required for prefix tuning.

        """
        batch_size = input_ids.shape[0]
        past_key_values = self.get_prompt(batch_size)
        fwd_params = list(inspect.signature(self.base_model.forward).parameters.keys())
        kwargs.update(
            {
                "input_ids": input_ids,
                "attention_mask": attention_mask,
                "inputs_embeds": inputs_embeds,
                "output_attentions": output_attentions,
                "output_hidden_states": output_hidden_states,
                "return_dict": return_dict,
                "past_key_values": past_key_values,
            }
        )
        if "past_key_values" in fwd_params:
            return self.base_model(labels=labels, **kwargs)
        else:
            transformer_backbone_name = self.base_model.get_subcell(self.transformer_backbone_name)
            fwd_params = list(inspect.signature(transformer_backbone_name.forward).parameters.keys())
            if "past_key_values" not in fwd_params:
                raise ValueError("Model does not support past key values which are required for prefix tuning.")
            outputs = transformer_backbone_name(**kwargs)
            sequence_output = outputs[0]
            if "dropout" in [name for name, _ in list(self.base_model.cells_and_names())]:
                sequence_output = self.base_model.dropout(sequence_output)
            logits = self.base_model.get_subcell(self.cls_layer_name)(sequence_output)

            loss = None
            if labels is not None:
                loss = ops.cross_entropy(logits.view(-1, self.num_labels), labels.view(-1))

            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

mindnlp.peft.peft_model.PeftModelForTokenClassification.__init__(model, peft_config=None, adapter_name='default')

Initializes a new instance of the PeftModelForTokenClassification class.

PARAMETER DESCRIPTION
self

The instance of the PeftModelForTokenClassification class.
model

The model used for token classification.
peft_config

The configuration for the Peft model. Defaults to None.

TYPE: PeftConfig DEFAULT: None

adapter_name

The name of the adapter. Defaults to 'default'.

TYPE: str DEFAULT: 'default'

RETURNS DESCRIPTION

None. This method does not return a value.
Source code in mindnlp/peft/peft_model.py 
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
def __init__(self, model, peft_config: PeftConfig = None, adapter_name="default"):
    r"""
    Initializes a new instance of the PeftModelForTokenClassification class.

    Args:
        self: The instance of the PeftModelForTokenClassification class.
        model: The model used for token classification.
        peft_config (PeftConfig, optional): The configuration for the Peft model. Defaults to None.
        adapter_name (str, optional): The name of the adapter. Defaults to 'default'.

    Returns:
        None. This method does not return a value.

    Raises:
        N/A
    """
    super().__init__(model, peft_config, adapter_name)
    if self.cells_to_save is None:
        self.cells_to_save = {"classifier", "score"}
    else:
        self.cells_to_save.update({"classifier", "score"})

    for name, _ in self.base_model.cells_and_names():
        if any(cell_name in name for cell_name in self.cells_to_save):
            self.cls_layer_name = name
            break

    # to make sure classifier layer is trainable
    _set_trainable(self, adapter_name)

mindnlp.peft.peft_model.PeftModelForTokenClassification.forward(input_ids=None, attention_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None, task_ids=None, **kwargs)

Forward pass of the model.

Source code in mindnlp/peft/peft_model.py 
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
def forward(
    self,
    input_ids=None,
    attention_mask=None,
    inputs_embeds=None,
    labels=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    task_ids=None,
    **kwargs,
):
    """
    Forward pass of the model.
    """
    peft_config = self.active_peft_config
    return_dict = return_dict if return_dict is not None else self.config.use_return_dict

    if not isinstance(peft_config, PromptLearningConfig):
        if peft_config.peft_type == PeftType.POLY:
            kwargs["task_ids"] = task_ids
        return self.base_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            inputs_embeds=inputs_embeds,
            labels=labels,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            **kwargs,
        )

    batch_size = input_ids.shape[0]
    if attention_mask is not None:
        # concat prompt attention mask
        prefix_attention_mask = ops.ones(batch_size, peft_config.num_virtual_tokens)
        attention_mask = ops.cat((prefix_attention_mask, attention_mask), axis=1)
    if kwargs.get("position_ids", None) is not None:
        warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.")
        kwargs["position_ids"] = None
    kwargs.update(
        {
            "attention_mask": attention_mask,
            "labels": labels,
            "output_attentions": output_attentions,
            "output_hidden_states": output_hidden_states,
            "return_dict": return_dict,
        }
    )

    if peft_config.peft_type == PeftType.PREFIX_TUNING:
        return self._prefix_tuning_forward(input_ids=input_ids, **kwargs)
    else:
        if kwargs.get("token_type_ids", None) is not None:
            kwargs["token_type_ids"] = ops.cat(
                (
                    ops.zeros(batch_size, peft_config.num_virtual_tokens),
                    kwargs["token_type_ids"],
                ),
                axis=1,
            ).long()
        if inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)
        prompts = self.get_prompt(batch_size=batch_size)
        prompts = prompts.to(inputs_embeds.dtype)
        inputs_embeds = ops.cat((prompts, inputs_embeds), axis=1)
        return self.base_model(inputs_embeds=inputs_embeds, **kwargs)