Class: NanoGPT::Layers::MLP

Inherits:
Torch::NN::Module
  • Object
show all
Defined in:
lib/nano_gpt/layers/mlp.rb

Overview

Feed-forward network with GELU activation

Instance Method Summary collapse

Constructor Details

#initialize(config) ⇒ MLP

Returns a new instance of MLP.



7
8
9
10
11
12
13
 
# File 'lib/nano_gpt/layers/mlp.rb', line 7

def initialize(config)
  super()
  @c_fc = Torch::NN::Linear.new(config.n_embd, 4 * config.n_embd, bias: config.bias)
  @gelu = Torch::NN::GELU.new
  @c_proj = Torch::NN::Linear.new(4 * config.n_embd, config.n_embd, bias: config.bias)
  @dropout = Torch::NN::Dropout.new(p: config.dropout)
end

Instance Method Details

#forward(x) ⇒ Object 



15
16
17
18
19
20
 
# File 'lib/nano_gpt/layers/mlp.rb', line 15

def forward(x)
  x = @c_fc.call(x)
  x = @gelu.call(x)
  x = @c_proj.call(x)
  @dropout.call(x)
end