Class: Transformers::DebertaV2::DebertaV2PreTrainedModel

Inherits:
PreTrainedModel
  • Object
show all
Defined in:
lib/transformers/models/deberta_v2/modeling_deberta_v2.rb

Instance Attribute Summary

Attributes inherited from PreTrainedModel

#config

Instance Method Summary collapse

Methods inherited from PreTrainedModel

#_backward_compatibility_gradient_checkpointing, #_initialize_weights, #base_model, #can_generate, #dequantize, #dummy_inputs, #framework, from_pretrained, #get_input_embeddings, #get_output_embeddings, #init_weights, #initialize, #post_init, #prune_heads, #set_input_embeddings, #tie_weights, #warn_if_padding_and_no_attention_mask

Methods included from ClassAttribute

#class_attribute

Methods included from ModuleUtilsMixin

#get_extended_attention_mask, #get_head_mask

Constructor Details

This class inherits a constructor from Transformers::PreTrainedModel

Instance Method Details

#_init_weights(module_) ⇒ Object

self._keys_to_ignore_on_load_unexpected = [“position_embeddings”] self.supports_gradient_checkpointing = true



710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
 
# File 'lib/transformers/models/deberta_v2/modeling_deberta_v2.rb', line 710

def _init_weights(module_)
  if module_.is_a?(Torch::NN::Linear)
    # Slightly different from the TF version which uses truncated_normal for initialization
    # cf https://github.com/pytorch/pytorch/pull/5617
    module_.weight.data.normal!(mean: 0.0, std: @config.initializer_range)
    if !module_.bias.nil?
      module_.bias.data.zero!
    end
  elsif module_.is_a?(Torch::NN::Embedding)
    module_.weight.data.normal!(mean: 0.0, std: @config.initializer_range)
    if !module_.padding_idx.nil?
      module_.weight.data.fetch(module_.padding_idx).zero!
    end
  end
end