Class: DNN::Layers::BatchNormalization

Inherits:
HasParamLayer show all
Defined in:
lib/dnn/core/layers.rb

Instance Attribute Summary collapse

Attributes inherited from HasParamLayer

#grads, #params

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from HasParamLayer

#update

Methods inherited from Layer

#built?, #prev_layer, #shape

Constructor Details

#initialize(momentum: 0.9, running_mean: nil, running_var: nil) ⇒ BatchNormalization

Returns a new instance of BatchNormalization.



518
519
520
521
522
523
 
# File 'lib/dnn/core/layers.rb', line 518

def initialize(momentum: 0.9, running_mean: nil, running_var: nil)
  super()
  @momentum = momentum
  @running_mean = running_mean
  @running_var = running_var
end

Instance Attribute Details

#momentum ⇒ Object (readonly)

Returns the value of attribute momentum.



516
517
518
 
# File 'lib/dnn/core/layers.rb', line 516

def momentum
  @momentum
end

Class Method Details

.load_hash(hash) ⇒ Object 



525
526
527
528
529
 
# File 'lib/dnn/core/layers.rb', line 525

def self.load_hash(hash)
  running_mean = SFloat.cast(hash[:running_mean])
  running_var = SFloat.cast(hash[:running_var])
  self.new(momentum: hash[:momentum], running_mean: running_mean, running_var: running_var)
end

Instance Method Details

#backward(dout) ⇒ Object 



554
555
556
557
558
559
560
561
562
563
564
565
 
# File 'lib/dnn/core/layers.rb', line 554

def backward(dout)
  batch_size = dout.shape[0]
  @grads[:beta] = dout.sum(0)
  @grads[:gamma] = (@xn * dout).sum(0)
  dxn = @params[:gamma] * dout
  dxc = dxn / @std
  dstd = -((dxn * @xc) / (@std**2)).sum(0)
  dvar = 0.5 * dstd / @std
  dxc += (2.0 / batch_size) * @xc * dvar
  dmean = dxc.sum(0)
  dxc - dmean / batch_size
end

#build(model) ⇒ Object 



531
532
533
534
535
 
# File 'lib/dnn/core/layers.rb', line 531

def build(model)
  super
  @running_mean ||= SFloat.zeros(*shape)
  @running_var ||= SFloat.zeros(*shape)
end

#forward(x) ⇒ Object 



537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
 
# File 'lib/dnn/core/layers.rb', line 537

def forward(x)
  if @model.training?
    mean = x.mean(0)
    @xc = x - mean
    var = (@xc**2).mean(0)
    @std = NMath.sqrt(var + 1e-7)
    xn = @xc / @std
    @xn = xn
    @running_mean = @momentum * @running_mean + (1 - @momentum) * mean
    @running_var = @momentum * @running_var + (1 - @momentum) * var
  else
    xc = x - @running_mean
    xn = xc / NMath.sqrt(@running_var + 1e-7)
  end
  @params[:gamma] * xn + @params[:beta]
end

#to_hash ⇒ Object 



567
568
569
570
571
 
# File 'lib/dnn/core/layers.rb', line 567

def to_hash
  super({momentum: @momentum,
         running_mean: @running_mean.to_a,
         running_var: @running_var.to_a})
end