Class: DNN::Layers::GRU_Dense

Inherits:
Object
  • Object
show all
Defined in:
lib/dnn/core/rnn_layers.rb

Instance Method Summary collapse

Constructor Details

#initialize(rnn) ⇒ GRU_Dense

Returns a new instance of GRU_Dense.



362
363
364
365
366
367
 
# File 'lib/dnn/core/rnn_layers.rb', line 362

def initialize(rnn)
  @rnn = rnn
  @update_sigmoid = Sigmoid.new
  @reset_sigmoid = Sigmoid.new
  @tanh = Tanh.new
end

Instance Method Details

#backward(dh2) ⇒ Object 



388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
 
# File 'lib/dnn/core/rnn_layers.rb', line 388

def backward(dh2)
  dtanh_h = @tanh.backward(dh2 * @update)
  dh = dh2 * (1 - @update)

  dweight_h = @x.transpose.dot(dtanh_h)
  dx = dtanh_h.dot(@weight_h.transpose)
  dweight2_h = (@h * @reset).transpose.dot(dtanh_h)
  dh += dtanh_h.dot(@weight2_h.transpose) * @reset
  dbias_h = dtanh_h.sum(0)

  dreset = @reset_sigmoid.backward(dtanh_h.dot(@weight2_h.transpose) * @h)
  dupdate = @update_sigmoid.backward(dh2 * @tanh_h - dh2 * @h)
  da = Xumo::SFloat.hstack([dupdate, dreset])
  dweight_a = @x.transpose.dot(da)
  dx += da.dot(@weight_a.transpose)
  dweight2_a = @h.transpose.dot(da)
  dh += da.dot(@weight2_a.transpose)
  dbias_a = da.sum(0)

  @rnn.grads[:weight] += Xumo::SFloat.hstack([dweight_a, dweight_h])
  @rnn.grads[:weight2] += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
  if @rnn.l1_lambda > 0
    @rnn.grads[:weight] += dlasso
    @rnn.grads[:weight2] += dlasso2
  elsif @rnn.l2_lambda > 0
    @rnn.grads[:weight] += dridge
    @rnn.grads[:weight2] += dridge2
  end
  @rnn.grads[:bias] += Xumo::SFloat.hstack([dbias_a, dbias_h])
  [dx, dh]
end

#forward(x, h) ⇒ Object 



369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
 
# File 'lib/dnn/core/rnn_layers.rb', line 369

def forward(x, h)
  @x = x
  @h = h
  num_nodes = h.shape[1]
  @weight_a = @rnn.params[:weight][true, 0...(num_nodes * 2)]
  @weight2_a = @rnn.params[:weight2][true, 0...(num_nodes * 2)]
  bias_a = @rnn.params[:bias][0...(num_nodes * 2)]
  a = x.dot(@weight_a) + h.dot(@weight2_a) + bias_a
  @update = @update_sigmoid.forward(a[true, 0...num_nodes])
  @reset = @reset_sigmoid.forward(a[true, num_nodes..-1])

  @weight_h = @rnn.params[:weight][true, (num_nodes * 2)..-1]
  @weight2_h = @rnn.params[:weight2][true, (num_nodes * 2)..-1]
  bias_h = @rnn.params[:bias][(num_nodes * 2)..-1]
  @tanh_h = @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
  h2 = (1 - @update) * h + @update * @tanh_h
  h2
end