Class: DNN::Layers::BatchNormalization

Inherits:
HasParamLayer show all
Defined in:
lib/dnn/core/layers.rb

Instance Attribute Summary collapse

Attributes inherited from HasParamLayer

#grads, #params, #trainable

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from HasParamLayer

#update

Methods inherited from Layer

#built?, #prev_layer, #shape

Constructor Details

#initialize(momentum: 0.9, running_mean: nil, running_var: nil) ⇒ BatchNormalization

Returns a new instance of BatchNormalization.



272
273
274
275
276
277
 
# File 'lib/dnn/core/layers.rb', line 272

def initialize(momentum: 0.9, running_mean: nil, running_var: nil)
  super()
  @momentum = momentum
  @running_mean = running_mean
  @running_var = running_var
end

Instance Attribute Details

#momentum ⇒ Object (readonly)

Returns the value of attribute momentum.



264
265
266
 
# File 'lib/dnn/core/layers.rb', line 264

def momentum
  @momentum
end

Class Method Details

.load_hash(hash) ⇒ Object 



266
267
268
269
270
 
# File 'lib/dnn/core/layers.rb', line 266

def self.load_hash(hash)
  running_mean = Xumo::SFloat.cast(hash[:running_mean])
  running_var = Xumo::SFloat.cast(hash[:running_var])
  self.new(momentum: hash[:momentum], running_mean: running_mean, running_var: running_var)
end

Instance Method Details

#backward(dout) ⇒ Object 



302
303
304
305
306
307
308
309
310
311
312
313
 
# File 'lib/dnn/core/layers.rb', line 302

def backward(dout)
  batch_size = dout.shape[0]
  @grads[:beta] = dout.sum(0)
  @grads[:gamma] = (@xn * dout).sum(0)
  dxn = @params[:gamma] * dout
  dxc = dxn / @std
  dstd = -((dxn * @xc) / (@std**2)).sum(0)
  dvar = 0.5 * dstd / @std
  dxc += (2.0 / batch_size) * @xc * dvar
  dmean = dxc.sum(0)
  dxc - dmean / batch_size
end

#build(model) ⇒ Object 



279
280
281
282
283
 
# File 'lib/dnn/core/layers.rb', line 279

def build(model)
  super
  @running_mean ||= Xumo::SFloat.zeros(*shape)
  @running_var ||= Xumo::SFloat.zeros(*shape)
end

#forward(x) ⇒ Object 



285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/dnn/core/layers.rb', line 285

def forward(x)
  if @model.training?
    mean = x.mean(0)
    @xc = x - mean
    var = (@xc**2).mean(0)
    @std = Xumo::NMath.sqrt(var + 1e-7)
    xn = @xc / @std
    @xn = xn
    @running_mean = @momentum * @running_mean + (1 - @momentum) * mean
    @running_var = @momentum * @running_var + (1 - @momentum) * var
  else
    xc = x - @running_mean
    xn = xc / Xumo::NMath.sqrt(@running_var + 1e-7)
  end
  @params[:gamma] * xn + @params[:beta]
end

#to_hash ⇒ Object 



315
316
317
318
319
 
# File 'lib/dnn/core/layers.rb', line 315

def to_hash
  super({momentum: @momentum,
         running_mean: @running_mean.to_a,
         running_var: @running_var.to_a})
end