Class: DNN::Optimizers::Adam

Inherits:
Optimizer show all
Includes:
Numo
Defined in:
lib/dnn/core/optimizers.rb

Instance Attribute Summary collapse

Attributes inherited from Optimizer

#learning_rate

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(learning_rate = 0.001, beta1 = 0.9, beta2 = 0.999) ⇒ Adam

Returns a new instance of Adam.



120
121
122
123
124
125
126
127
 
# File 'lib/dnn/core/optimizers.rb', line 120

def initialize(learning_rate = 0.001, beta1 = 0.9, beta2 = 0.999)
  super(learning_rate)
  @beta1 = beta1
  @beta2 = beta2
  @iter = 0
  @m = {}
  @v = {}
end

Instance Attribute Details

#beta1 ⇒ Object

Returns the value of attribute beta1.



117
118
119
 
# File 'lib/dnn/core/optimizers.rb', line 117

def beta1
  @beta1
end

#beta2 ⇒ Object

Returns the value of attribute beta2.



118
119
120
 
# File 'lib/dnn/core/optimizers.rb', line 118

def beta2
  @beta2
end

Class Method Details

.load_hash(hash) ⇒ Object 



129
130
131
 
# File 'lib/dnn/core/optimizers.rb', line 129

def self.load_hash(hash)
  self.new(hash[:learning_rate], hash[:beta1], hash[:beta2])
end

Instance Method Details

#to_hash ⇒ Object 



147
148
149
150
151
152
153
154
 
# File 'lib/dnn/core/optimizers.rb', line 147

def to_hash
  {
    name: self.class.name,
    learning_rate: @learning_rate,
    beta1: @beta1,
    beta2: @beta2,
  }
end

#update(layer) ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/dnn/core/optimizers.rb', line 133

def update(layer)
  @iter += 1
  @m[layer] ||= {}
  @v[layer] ||= {}
  lr = @learning_rate * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
  layer.params.each_key do |key|
    @m[layer][key] ||= 0
    @v[layer][key] ||= 0
    @m[layer][key] += (1 - @beta1) * (layer.grads[key] - @m[layer][key])
    @v[layer][key] += (1 - @beta2) * (layer.grads[key]**2 - @v[layer][key])
    layer.params[key] -= lr * @m[layer][key] / NMath.sqrt(@v[layer][key] + 1e-7)
  end
end