Class: Transformers::PretrainedConfig

Inherits:
Object
  • Object
show all
Extended by:
ClassAttribute
Defined in:
lib/transformers/configuration_utils.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ClassAttribute

class_attribute

Constructor Details

#initialize(**kwargs) ⇒ PretrainedConfig

Returns a new instance of PretrainedConfig.



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/transformers/configuration_utils.rb', line 41

def initialize(**kwargs)
  @return_dict = kwargs.delete(:return_dict) { true }
  @output_hidden_states = kwargs.delete(:output_hidden_states) { false }
  @output_attentions = kwargs.delete(:output_attentions) { false }
  @pruned_heads = kwargs.delete(:pruned_heads) { {} }
  @tie_word_embeddings = kwargs.delete(:tie_word_embeddings) { true }
  @chunk_size_feed_forward = kwargs.delete(:chunk_size_feed_forward) { 0 }

  # Is decoder is used in encoder-decoder models to differentiate encoder from decoder
  @is_encoder_decoder = kwargs.delete(:is_encoder_decoder) { false }
  @is_decoder = kwargs.delete(:is_decoder) { false }
  @cross_attention_hidden_size = kwargs.delete(:cross_attention_hidden_size)
  @add_cross_attention = kwargs.delete(:add_cross_attention) { false }
  @tie_encoder_decoder = kwargs.delete(:tie_encoder_decoder) { false }

  # Fine-tuning task arguments
  @architectures = kwargs.delete(:architectures)
  @finetuning_task = kwargs.delete(:finetuning_task)
  @id2label = kwargs.delete(:id2label)
  @label2id = kwargs.delete(:label2id)
  if !@label2id.nil? && !@label2id.is_a?(Hash)
    raise ArgumentError, "Argument label2id should be a dictionary."
  end
  if !@id2label.nil?
    if !@id2label.is_a?(Hash)
      raise ArgumentError, "Argument id2label should be a dictionary."
    end
    num_labels = kwargs.delete(:num_labels)
    if !num_labels.nil? && id2label.length != num_labels
      raise Todo
    end
    @id2label = @id2label.transform_keys(&:to_i)
    # Keys are always strings in JSON so convert ids to int here.
  else
    self.num_labels = kwargs.delete(:num_labels) { 2 }
  end

  # Tokenizer arguments TODO: eventually tokenizer and models should share the same config
  @tokenizer_class = kwargs.delete(:tokenizer_class)
  @prefix = kwargs.delete(:prefix)
  @bos_token_id = kwargs.delete(:bos_token_id)
  @pad_token_id = kwargs.delete(:pad_token_id)
  @eos_token_id = kwargs.delete(:eos_token_id)
  @sep_token_id = kwargs.delete(:sep_token_id)

  # regression / multi-label classification
  @problem_type = kwargs.delete(:problem_type)

  # Name or path to the pretrained checkpoint
  @name_or_path = kwargs.delete(:name_or_path).to_s
  # Config hash
  @commit_hash = kwargs.delete(:_commit_hash)

  # TODO set kwargs
  @gradient_checkpointing = kwargs[:gradient_checkpointing]
  @output_past = kwargs[:output_past]
  @tie_weights_ = kwargs[:tie_weights_]
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(m, *args, **kwargs) ⇒ Object

TODO support setter



24
25
26
27
28
29
30
 
# File 'lib/transformers/configuration_utils.rb', line 24

def method_missing(m, *args, **kwargs)
  if self.class.attribute_map.include?(m)
    instance_variable_get("@#{self.class.attribute_map[m]}")
  else
    super
  end
end

Instance Attribute Details

#_commit_hash ⇒ Object (readonly)

Returns the value of attribute _commit_hash.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def _commit_hash
  @_commit_hash
end

#add_cross_attention ⇒ Object (readonly)

Returns the value of attribute add_cross_attention.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def add_cross_attention
  @add_cross_attention
end

#architectures ⇒ Object (readonly)

Returns the value of attribute architectures.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def architectures
  @architectures
end

#chunk_size_feed_forward ⇒ Object (readonly)

Returns the value of attribute chunk_size_feed_forward.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def chunk_size_feed_forward
  @chunk_size_feed_forward
end

#id2label ⇒ Object (readonly)

Returns the value of attribute id2label.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def id2label
  @id2label
end

#is_decoder ⇒ Object (readonly)

Returns the value of attribute is_decoder.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def is_decoder
  @is_decoder
end

#is_encoder_decoder ⇒ Object (readonly)

Returns the value of attribute is_encoder_decoder.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def is_encoder_decoder
  @is_encoder_decoder
end

#output_attentions ⇒ Object (readonly)

Returns the value of attribute output_attentions.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def output_attentions
  @output_attentions
end

#output_hidden_states ⇒ Object (readonly)

Returns the value of attribute output_hidden_states.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def output_hidden_states
  @output_hidden_states
end

#pad_token_id ⇒ Object (readonly)

Returns the value of attribute pad_token_id.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def pad_token_id
  @pad_token_id
end

#problem_type ⇒ Object (readonly)

Returns the value of attribute problem_type.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def problem_type
  @problem_type
end

#pruned_heads ⇒ Object (readonly)

Returns the value of attribute pruned_heads.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def pruned_heads
  @pruned_heads
end

#tie_encoder_decoder ⇒ Object (readonly)

Returns the value of attribute tie_encoder_decoder.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def tie_encoder_decoder
  @tie_encoder_decoder
end

#tie_word_embeddings ⇒ Object (readonly)

Returns the value of attribute tie_word_embeddings.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def tie_word_embeddings
  @tie_word_embeddings
end

#tokenizer_class ⇒ Object (readonly)

Returns the value of attribute tokenizer_class.



37
38
39
 
# File 'lib/transformers/configuration_utils.rb', line 37

def tokenizer_class
  @tokenizer_class
end

Class Method Details

.from_dict(config_dict, **kwargs) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/transformers/configuration_utils.rb', line 200

def from_dict(config_dict, **kwargs)
  return_unused_kwargs = kwargs.delete(:return_unused_kwargs) { false }

  # The commit hash might have been updated in the `config_dict`, we don't want the kwargs to erase that update.
  if kwargs.include?(:_commit_hash) && config_dict.include?(:_commit_hash)
    kwargs[:_commit_hash] = config_dict[:_commit_hash]
  end

  config = new(**config_dict)

  kwargs.each do |key, value|
    if config.respond_to?("#{key}=")
      config.public_send("#{key}=", value)
    end
  end

  Transformers.logger.info("Model config #{config}")
  if return_unused_kwargs
    [config, kwargs]
  else
    config
  end
end

.from_pretrained(pretrained_model_name_or_path, cache_dir: nil, force_download: false, local_files_only: false, token: nil, revision: "main", **kwargs) ⇒ Object 



186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/transformers/configuration_utils.rb', line 186

def from_pretrained(
  pretrained_model_name_or_path,
  cache_dir: nil,
  force_download: false,
  local_files_only: false,
  token: nil,
  revision: "main",
  **kwargs
)
  config_dict, kwargs = get_config_dict(pretrained_model_name_or_path, **kwargs)

  from_dict(config_dict, **kwargs)
end

.get_config_dict(pretrained_model_name_or_path, **kwargs) ⇒ Object 



224
225
226
227
228
229
 
# File 'lib/transformers/configuration_utils.rb', line 224

def get_config_dict(pretrained_model_name_or_path, **kwargs)
  # Get config dict associated with the base config file
  config_dict, kwargs = _get_config_dict(pretrained_model_name_or_path, **kwargs)

  [config_dict, kwargs]
end

Instance Method Details

#_attn_implementation ⇒ Object 



119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/transformers/configuration_utils.rb', line 119

def _attn_implementation
  # This property is made private for now (as it cannot be changed and a PreTrainedModel.use_attn_implementation method needs to be implemented.)
  if instance_variable_defined?(:@attn_implementation_internal)
    if instance_variable_get(:@attn_implementation_internal).nil?
      # `config.attn_implementation` should never be None, for backward compatibility.
      "eager"
    else
      @attn_implementation_internal
    end
  else
    "eager"
  end
end

#_dict ⇒ Object 



159
160
161
 
# File 'lib/transformers/configuration_utils.rb', line 159

def _dict
  instance_variables.to_h { |k| [k[1..].to_sym, instance_variable_get(k)] }
end

#name_or_path ⇒ Object 



100
101
102
 
# File 'lib/transformers/configuration_utils.rb', line 100

def name_or_path
  @name_or_path
end

#name_or_path=(value) ⇒ Object 



104
105
106
 
# File 'lib/transformers/configuration_utils.rb', line 104

def name_or_path=(value)
  @name_or_path = value.to_s
end

#num_labels ⇒ Object 



108
109
110
 
# File 'lib/transformers/configuration_utils.rb', line 108

def num_labels
  @id2label.length
end

#num_labels=(num_labels) ⇒ Object 



112
113
114
115
116
117
 
# File 'lib/transformers/configuration_utils.rb', line 112

def num_labels=(num_labels)
  if @id2label.nil? || @id2label.length != num_labels
    @id2label = num_labels.times.to_h { |i| [i, "LABEL_#{i}"] }
    @label2id =  @id2label.invert
  end
end

#respond_to_missing?(m, include_private = true) ⇒ Boolean

TODO support setter

Returns:

  • (Boolean) 


33
34
35
 
# File 'lib/transformers/configuration_utils.rb', line 33

def respond_to_missing?(m, include_private = true)
  self.class.attribute_map.include?(m) || super
end

#to_dict ⇒ Object 



163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/transformers/configuration_utils.rb', line 163

def to_dict
  output = Copy.deepcopy(_dict)
  output[:model_type] = self.class.model_type
  output.delete(:_auto_class)
  output.delete(:_commit_hash)
  output.delete(:_attn_implementation_internal)

  # Transformers version when serializing the model
  output[:transformers_version] = VERSION

  output
end

#to_diff_dict ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/transformers/configuration_utils.rb', line 141

def to_diff_dict
  config_dict = to_dict

  # get the default config dict
  default_config_dict = PretrainedConfig.new.to_dict

  serializable_config_dict = {}

  config_dict.each do |key, value|
    key = :_name_or_path if key == :name_or_path
    if !default_config_dict.include?(key) || value != default_config_dict[key] || key == :transformers_version
      serializable_config_dict[key] = value
    end
  end

  serializable_config_dict
end

#to_json_string(use_diff: true) ⇒ Object 



176
177
178
179
180
181
182
183
 
# File 'lib/transformers/configuration_utils.rb', line 176

def to_json_string(use_diff: true)
  if use_diff == true
    config_dict = to_diff_dict
  else
    config_dict = to_dict
  end
  JSON.pretty_generate(config_dict.sort_by { |k, _| k }.to_h) + "\n"
end

#to_s ⇒ Object 



137
138
139
 
# File 'lib/transformers/configuration_utils.rb', line 137

def to_s
  "#{self.class.name} #{to_json_string}"
end

#use_return_dict ⇒ Object 



133
134
135
 
# File 'lib/transformers/configuration_utils.rb', line 133

def use_return_dict
  @return_dict
end