Class: DNN::Optimizers::AdaDelta

Inherits:
Optimizer show all
Defined in:
lib/dnn/core/optimizers.rb

Instance Attribute Summary collapse

Attributes inherited from Optimizer

#learning_rate

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(rho: 0.95) ⇒ AdaDelta

Returns a new instance of AdaDelta.



132
133
134
135
136
137
 
# File 'lib/dnn/core/optimizers.rb', line 132

def initialize(rho: 0.95)
  super(nil)
  @rho = rho
  @h = {}
  @s = {}
end

Instance Attribute Details

#rho ⇒ Object

Returns the value of attribute rho.



126
127
128
 
# File 'lib/dnn/core/optimizers.rb', line 126

def rho
  @rho
end

Class Method Details

.load_hash(hash) ⇒ Object 



128
129
130
 
# File 'lib/dnn/core/optimizers.rb', line 128

def self.load_hash(hash)
  self.new(rho: hash[:rho])
end

Instance Method Details

#to_hash ⇒ Object 



152
153
154
 
# File 'lib/dnn/core/optimizers.rb', line 152

def to_hash
  super({rho: @rho})
end

#update(layer) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
 
# File 'lib/dnn/core/optimizers.rb', line 139

def update(layer)
  @h[layer] ||= {}
  @s[layer] ||= {}
  layer.params.each_key do |key|
    @h[layer][key] ||= Xumo::SFloat.zeros(*layer.params[key].shape)
    @s[layer][key] ||= Xumo::SFloat.zeros(*layer.params[key].shape)
    @h[layer][key] = @rho * @h[layer][key] + (1 - @rho) * layer.grads[key]**2
    v = (Xumo::NMath.sqrt(@s[layer][key] + 1e-6) / Xumo::NMath.sqrt(@h[layer][key] + 1e-6)) * layer.grads[key]
    @s[layer][key] = @rho * @s[layer][key] + (1 - @rho) * v**2
    layer.params[key] -= v
  end
end