Skip to content

Adaption_Prompt

mindnlp.peft.tuners.adaption_prompt.config.AdaptionPromptConfig dataclass

Bases: PeftConfig

Stores the configuration of an [AdaptionPromptModel].

Source code in mindnlp/peft/tuners/adaption_prompt/config.py 
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
@dataclass
class AdaptionPromptConfig(PeftConfig):
    """Stores the configuration of an [`AdaptionPromptModel`]."""
    target_cells: str = field(
        default=None, metadata={"help": "Name of the attention subcells to insert adaption prompts into."}
    )
    adapter_len: int = field(default=None, metadata={"help": "Number of adapter tokens to insert"})
    adapter_layers: int = field(default=None, metadata={"help": "Number of adapter layers (from the top)"})

    def __post_init__(self):
        r"""
        This method is called automatically after the initialization of an instance of the 'AdaptionPromptConfig' class.

        Args:
            self: An instance of the 'AdaptionPromptConfig' class.

        Returns:
            None. This method does not return any value.

        Raises:
            None.

        Description:
        This method sets the 'peft_type' attribute of the 'AdaptionPromptConfig' instance to 'PeftType.ADAPTION_PROMPT'.
        The 'peft_type' attribute represents the type of the adaption prompt configuration.

        Example:
            config = AdaptionPromptConfig()
            config.__post_init__()
            print(config.peft_type)  # Output: PeftType.ADAPTION_PROMPT
        """
        self.peft_type = PeftType.ADAPTION_PROMPT

    @property
    def is_adaption_prompt(self) -> bool:
        """Return True if this is an adaption prompt config."""
        return True

mindnlp.peft.tuners.adaption_prompt.config.AdaptionPromptConfig.is_adaption_prompt: bool property

Return True if this is an adaption prompt config.

mindnlp.peft.tuners.adaption_prompt.config.AdaptionPromptConfig.__post_init__()

This method is called automatically after the initialization of an instance of the 'AdaptionPromptConfig' class.

PARAMETER DESCRIPTION
self

An instance of the 'AdaptionPromptConfig' class.
RETURNS DESCRIPTION

None. This method does not return any value.

This method sets the 'peft_type' attribute of the 'AdaptionPromptConfig' instance to 'PeftType.ADAPTION_PROMPT'. The 'peft_type' attribute represents the type of the adaption prompt configuration.

Example

config = AdaptionPromptConfig() config.post_init() print(config.peft_type) # Output: PeftType.ADAPTION_PROMPT

Source code in mindnlp/peft/tuners/adaption_prompt/config.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def __post_init__(self):
    r"""
    This method is called automatically after the initialization of an instance of the 'AdaptionPromptConfig' class.

    Args:
        self: An instance of the 'AdaptionPromptConfig' class.

    Returns:
        None. This method does not return any value.

    Raises:
        None.

    Description:
    This method sets the 'peft_type' attribute of the 'AdaptionPromptConfig' instance to 'PeftType.ADAPTION_PROMPT'.
    The 'peft_type' attribute represents the type of the adaption prompt configuration.

    Example:
        config = AdaptionPromptConfig()
        config.__post_init__()
        print(config.peft_type)  # Output: PeftType.ADAPTION_PROMPT
    """
    self.peft_type = PeftType.ADAPTION_PROMPT

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel

Bases: Module

Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.

The top L attention cells are replaced with AdaptedAttention cells that wrap the original ones, but insert trainable prompts with gates (for zero init).

Notes on the multi-adapter pattern: - We store the states of different adapters by keeping a dictionary of AdaptedAttention cells indexed by adapter name. - Every time we switch adapters, we remove the cells of the currently active adapter from the model, store them in the dictionary, and replace them with the cells of the new adapter. - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the dictionary. - Disabling the adapter would also result in the cells being removed from the model.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
class AdaptionPromptModel(nn.Module):
    """
    Implements adaption prompts as described in https://arxiv.org/pdf/2303.16199.pdf.

    The top L attention cells are replaced with AdaptedAttention cells that wrap the original ones, but insert
    trainable prompts with gates (for zero init).

    Notes on the multi-adapter pattern:
    - We store the states of different adapters by keeping a dictionary of AdaptedAttention cells indexed by adapter
      name.
    - Every time we switch adapters, we remove the cells of the currently active adapter from the model, store them
      in the dictionary, and replace them with the cells of the new adapter.
    - To avoid duplicated and potentially inconsistent state, the currently active adapter is always removed from the
      dictionary.
    - Disabling the adapter would also result in the cells being removed from the model.
    """
    def __init__(self, model, configs: Dict, adapter_name: str):
        r"""
        Initializes an instance of the AdaptionPromptModel class.

        Args:
            self: The current instance of the class.
            model: The underlying model to be used for adaption prompts. Expected to be an object of a specific model class.
            configs: A dictionary containing configuration details for the adaption prompt model.
                - Type: Dict
                - Purpose: Specifies various configurations required for the adaption prompt model.
                - Restrictions: None
            adapter_name: The name of the adapter to be added.
                - Type: str
                - Purpose: Identifies the adapter which needs to be added to the adaption prompt model.
                - Restrictions: None

        Returns:
            None

        Raises:
            None
        """
        super(AdaptionPromptModel, self).__init__()
        self.model = model
        self.peft_config = {}
        self._parents = {}
        self._cached_adapters = {}
        self._active_adapter = None
        self._enabled = True
        self.forward = self.model.forward
        self.add_adapter(adapter_name, configs[adapter_name])
        self._mark_only_adaption_prompts_as_trainable(self.model)

    def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
        """Add an adapter with the given name and config."""
        config = prepare_config(config, self.model)
        if adapter_name in self.peft_config:
            raise ValueError(f"Adapter named '{adapter_name}' already exists.")

        parents = []
        # 获取模型的所有子模块及其名称
        for name, subcell in self.model.name_cells().items():
            if name.endswith(config.target_cells):
                # 对每个符合条件的子模块调用 _get_subcells 函数
                parent, target, target_name = _get_subcells(self.model, name)
                if target == subcell:
                    parents.append(parent)

        if len(parents) < config.adapter_layers:
            raise ValueError("Config specifies more adapter layers than available in the model.")

        parents = parents[-config.adapter_layers:]
        self._parents[adapter_name] = parents

        if self._active_adapter and self._enabled:
            self._remove_adapted_attentions(self._active_adapter)
        self._active_adapter = adapter_name
        self.peft_config[adapter_name] = config
        self._create_adapted_attentions(config, parents)
        if not self._enabled:
            self._remove_adapted_attentions(adapter_name)

        if config.inference_mode:
            _freeze_adapter(self.model, adapter_name)

    def set_adapter(self, adapter_name: str) -> None:
        """Set the model to use the adapter with the given name."""
        if self._active_adapter == adapter_name:
            return
        if adapter_name not in self.peft_config:
            raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")

        if self._enabled:
            self._remove_adapted_attentions(self._active_adapter)
            self._set_adapted_attentions(adapter_name)

        self._active_adapter = adapter_name

    def enable_adapter_layers(self):
        """Enable adapter layers by swapping in cached AdaptedAttention cells."""
        self._enabled = True
        self._set_adapted_attentions(self._active_adapter)

    def disable_adapter_layers(self):
        """Disable adapter layers by swapping out AdaptedAttention cells."""
        self._enabled = False
        self._remove_adapted_attentions(self._active_adapter)

    def _create_adapted_attentions(self, config: AdaptionPromptConfig, parents: List[nn.Module]) -> None:
        """Wrap LlamaAttention cells with newly created AdaptedAttention cells."""
        for par in parents:
            attn = AdaptedAttention(
                model_type=self.model.config.model_type,
                adapter_len=config.adapter_len,
                model=getattr(par, config.target_cells),
            )
            setattr(par, config.target_cells, attn)

    def _set_adapted_attentions(self, adapter_name: str) -> None:
        """Replace LlamaAttention cells with cached AdaptedAttention cells."""
        cached = self._cached_adapters[adapter_name]
        del self._cached_adapters[adapter_name]
        config = self.peft_config[adapter_name]
        for i, par in enumerate(self._parents[adapter_name]):
            setattr(par, config.target_cells, cached[i])

    def _remove_adapted_attentions(self, adapter_name: str) -> None:
        """Remove AdaptedAttention cells from the model and store them in the cache."""
        config = self.peft_config[adapter_name]
        adapted_attentions = []
        for par in self._parents[adapter_name]:
            attn = getattr(par, config.target_cells)
            adapted_attentions.append(attn)
            setattr(par, config.target_cells, attn.model)
        self._cached_adapters[adapter_name] = adapted_attentions

    def _mark_only_adaption_prompts_as_trainable(self, model: nn.Module) -> None:
        r"""Marks only adaption prompts as trainable in the given model.

        Args:
            self (AdaptionPromptModel): The instance of AdaptionPromptModel class.
            model (nn.Module): The model for which adaption prompts need to be marked as trainable.

        Returns:
            None. This method does not return any value.

        Raises:
            None.
        """
        for param in model.trainable_params():
            if not is_adaption_prompt_trainable(param.name):
                param.requires_grad = False
    def __getattr__(self, name: str):
        """Forward missing attributes to the wrapped cell."""
        try:
            return super().__getattr__(name)  # defer to nn.Module's logic
        except AttributeError:
            # This is necessary as e.g. causal models have various methods that we
            # don't want to re-implement here.
            return getattr(self.model, name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.__getattr__(name)

Forward missing attributes to the wrapped cell.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
174
175
176
177
178
179
180
181
def __getattr__(self, name: str):
    """Forward missing attributes to the wrapped cell."""
    try:
        return super().__getattr__(name)  # defer to nn.Module's logic
    except AttributeError:
        # This is necessary as e.g. causal models have various methods that we
        # don't want to re-implement here.
        return getattr(self.model, name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.__init__(model, configs, adapter_name)

Initializes an instance of the AdaptionPromptModel class.

PARAMETER DESCRIPTION
self

The current instance of the class.
model

The underlying model to be used for adaption prompts. Expected to be an object of a specific model class.
configs

A dictionary containing configuration details for the adaption prompt model. - Type: Dict - Purpose: Specifies various configurations required for the adaption prompt model. - Restrictions: None

TYPE: Dict

adapter_name

The name of the adapter to be added. - Type: str - Purpose: Identifies the adapter which needs to be added to the adaption prompt model. - Restrictions: None

TYPE: str

RETURNS DESCRIPTION

None
Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def __init__(self, model, configs: Dict, adapter_name: str):
    r"""
    Initializes an instance of the AdaptionPromptModel class.

    Args:
        self: The current instance of the class.
        model: The underlying model to be used for adaption prompts. Expected to be an object of a specific model class.
        configs: A dictionary containing configuration details for the adaption prompt model.
            - Type: Dict
            - Purpose: Specifies various configurations required for the adaption prompt model.
            - Restrictions: None
        adapter_name: The name of the adapter to be added.
            - Type: str
            - Purpose: Identifies the adapter which needs to be added to the adaption prompt model.
            - Restrictions: None

    Returns:
        None

    Raises:
        None
    """
    super(AdaptionPromptModel, self).__init__()
    self.model = model
    self.peft_config = {}
    self._parents = {}
    self._cached_adapters = {}
    self._active_adapter = None
    self._enabled = True
    self.forward = self.model.forward
    self.add_adapter(adapter_name, configs[adapter_name])
    self._mark_only_adaption_prompts_as_trainable(self.model)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.add_adapter(adapter_name, config)

Add an adapter with the given name and config.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def add_adapter(self, adapter_name: str, config: AdaptionPromptConfig) -> None:
    """Add an adapter with the given name and config."""
    config = prepare_config(config, self.model)
    if adapter_name in self.peft_config:
        raise ValueError(f"Adapter named '{adapter_name}' already exists.")

    parents = []
    # 获取模型的所有子模块及其名称
    for name, subcell in self.model.name_cells().items():
        if name.endswith(config.target_cells):
            # 对每个符合条件的子模块调用 _get_subcells 函数
            parent, target, target_name = _get_subcells(self.model, name)
            if target == subcell:
                parents.append(parent)

    if len(parents) < config.adapter_layers:
        raise ValueError("Config specifies more adapter layers than available in the model.")

    parents = parents[-config.adapter_layers:]
    self._parents[adapter_name] = parents

    if self._active_adapter and self._enabled:
        self._remove_adapted_attentions(self._active_adapter)
    self._active_adapter = adapter_name
    self.peft_config[adapter_name] = config
    self._create_adapted_attentions(config, parents)
    if not self._enabled:
        self._remove_adapted_attentions(adapter_name)

    if config.inference_mode:
        _freeze_adapter(self.model, adapter_name)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.disable_adapter_layers()

Disable adapter layers by swapping out AdaptedAttention cells.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
125
126
127
128
def disable_adapter_layers(self):
    """Disable adapter layers by swapping out AdaptedAttention cells."""
    self._enabled = False
    self._remove_adapted_attentions(self._active_adapter)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.enable_adapter_layers()

Enable adapter layers by swapping in cached AdaptedAttention cells.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
120
121
122
123
def enable_adapter_layers(self):
    """Enable adapter layers by swapping in cached AdaptedAttention cells."""
    self._enabled = True
    self._set_adapted_attentions(self._active_adapter)

mindnlp.peft.tuners.adaption_prompt.model.AdaptionPromptModel.set_adapter(adapter_name)

Set the model to use the adapter with the given name.

Source code in mindnlp/peft/tuners/adaption_prompt/model.py 
107
108
109
110
111
112
113
114
115
116
117
118
def set_adapter(self, adapter_name: str) -> None:
    """Set the model to use the adapter with the given name."""
    if self._active_adapter == adapter_name:
        return
    if adapter_name not in self.peft_config:
        raise ValueError(f"Adapter with name '{adapter_name}' does not exist.")

    if self._enabled:
        self._remove_adapted_attentions(self._active_adapter)
        self._set_adapted_attentions(adapter_name)

    self._active_adapter = adapter_name