Class: DNN::Optimizers::Adam

Inherits:
Optimizer show all
Defined in:
lib/dnn/core/optimizers.rb

Instance Attribute Summary collapse

Attributes inherited from Optimizer

#learning_rate

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, eps: 1e-7) ⇒ Adam

Returns a new instance of Adam.

Parameters:

  • alpha (Float) (defaults to: 0.001) —

    Value used to calculate learning rate.

  • beta1 (Float) (defaults to: 0.9) —

    Moving average index of beta1.

  • beta2 (Float) (defaults to: 0.999) —

    Moving average index of beta2.

  • eps (Float) (defaults to: 1e-7) —

    Value to avoid division by zero.



211
212
213
214
215
216
217
218
219
220
 
# File 'lib/dnn/core/optimizers.rb', line 211

def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, eps: 1e-7)
  super(nil)
  @alpha = alpha
  @beta1 = beta1
  @beta2 = beta2
  @eps = eps
  @iter = 0
  @m = {}
  @v = {}
end

Instance Attribute Details

#alpha ⇒ Float

Return the alpha value.

Returns:

  • (Float) —

    Return the alpha value.



195
196
197
 
# File 'lib/dnn/core/optimizers.rb', line 195

def alpha
  @alpha
end

#beta1 ⇒ Float

Return the beta1 value.

Returns:

  • (Float) —

    Return the beta1 value.



197
198
199
 
# File 'lib/dnn/core/optimizers.rb', line 197

def beta1
  @beta1
end

#beta2 ⇒ Float

Return the beta2 value.

Returns:

  • (Float) —

    Return the beta2 value.



199
200
201
 
# File 'lib/dnn/core/optimizers.rb', line 199

def beta2
  @beta2
end

#eps ⇒ Float

Return the eps value.

Returns:

  • (Float) —

    Return the eps value.



201
202
203
 
# File 'lib/dnn/core/optimizers.rb', line 201

def eps
  @eps
end

Class Method Details

.from_hash(hash) ⇒ Object 



203
204
205
 
# File 'lib/dnn/core/optimizers.rb', line 203

def self.from_hash(hash)
  self.new(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2], eps: hash[:eps])
end

Instance Method Details

#to_hash ⇒ Object 



234
235
236
 
# File 'lib/dnn/core/optimizers.rb', line 234

def to_hash
  super(alpha: @alpha, beta1: @beta1, beta2: @beta2, eps: @eps)
end

#update(layers) ⇒ Object 



222
223
224
225
226
227
228
229
230
231
232
 
# File 'lib/dnn/core/optimizers.rb', line 222

def update(layers)
  @iter += 1
  learning_rate = @alpha * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
  target_params = layers.select { |layer| layer.is_a?(HasParamLayer) && layer.trainable }
                        .map { |layer| layer.params.values }.flatten
                        .select { |param| param.grad }
  target_params.each do |param|
    update_param(param, learning_rate)
    param.grad = 0
  end
end