Class: DNN::Layers::BatchNormalization

Inherits:
HasParamLayer show all
Defined in:
lib/dnn/core/layers.rb

Instance Attribute Summary collapse

Attributes inherited from HasParamLayer

#params, #trainable

Attributes inherited from Layer

#input_shape

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from HasParamLayer

#build, #update

Methods inherited from Layer

#build, #built?, #output_shape

Constructor Details

#initialize(momentum: 0.9) ⇒ BatchNormalization

Returns a new instance of BatchNormalization.

Parameters:

  • momentum (Float) (defaults to: 0.9) —

    Exponential moving average of mean and variance.



326
327
328
329
 
# File 'lib/dnn/core/layers.rb', line 326

def initialize(momentum: 0.9)
  super()
  @momentum = momentum
end

Instance Attribute Details

#momentum ⇒ Float (readonly)

Returns Exponential moving average of mean and variance.

Returns:

  • (Float) —

    Exponential moving average of mean and variance.



319
320
321
 
# File 'lib/dnn/core/layers.rb', line 319

def momentum
  @momentum
end

Class Method Details

.load_hash(hash) ⇒ Object 



321
322
323
 
# File 'lib/dnn/core/layers.rb', line 321

def self.load_hash(hash)
  self.new(momentum: hash[:momentum])
end

Instance Method Details

#backward(dout) ⇒ Object 



348
349
350
351
352
353
354
355
356
357
358
359
 
# File 'lib/dnn/core/layers.rb', line 348

def backward(dout)
  batch_size = dout.shape[0]
  @beta.grad = dout.sum(0)
  @gamma.grad = (@xn * dout).sum(0)
  dxn = @gamma.data * dout
  dxc = dxn / @std
  dstd = -((dxn * @xc) / (@std**2)).sum(0)
  dvar = 0.5 * dstd / @std
  dxc += (2.0 / batch_size) * @xc * dvar
  dmean = dxc.sum(0)
  dxc - dmean / batch_size
end

#forward(x, learning_phase) ⇒ Object 



331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
 
# File 'lib/dnn/core/layers.rb', line 331

def forward(x, learning_phase)
  if learning_phase
    mean = x.mean(0)
    @xc = x - mean
    var = (@xc**2).mean(0)
    @std = NMath.sqrt(var + 1e-7)
    xn = @xc / @std
    @xn = xn
    @running_mean.data = @momentum * @running_mean.data + (1 - @momentum) * mean
    @running_var.data = @momentum * @running_var.data + (1 - @momentum) * var
  else
    xc = x - @running_mean.data
    xn = xc / NMath.sqrt(@running_var.data + 1e-7)
  end
  @gamma.data * xn + @beta.data
end

#to_hash ⇒ Object 



361
362
363
 
# File 'lib/dnn/core/layers.rb', line 361

def to_hash
  super({momentum: @momentum})
end