Method: Transformers::Distilbert::FFN#initialize

Defined in:
lib/transformers/models/distilbert/modeling_distilbert.rb

#initialize(config) ⇒ FFN

Returns a new instance of FFN.



154
155
156
157
158
159
160
161
162
 
# File 'lib/transformers/models/distilbert/modeling_distilbert.rb', line 154

def initialize(config)
  super()
  @dropout = Torch::NN::Dropout.new(p: config.dropout)
  @chunk_size_feed_forward = config.chunk_size_feed_forward
  @seq_len_dim = 1
  @lin1 = Torch::NN::Linear.new(config.dim, config.hidden_dim)
  @lin2 = Torch::NN::Linear.new(config.hidden_dim, config.dim)
  @activation = Activations.get_activation(config.activation)
end