Class: DNN::Layers::GRU_Dense

Inherits:
Object
  • Object
show all
Defined in:
lib/dnn/core/rnn_layers.rb

Instance Method Summary collapse

Constructor Details

#initialize(rnn) ⇒ GRU_Dense

Returns a new instance of GRU_Dense.



357
358
359
360
361
362
 
# File 'lib/dnn/core/rnn_layers.rb', line 357

def initialize(rnn)
  @rnn = rnn
  @update_sigmoid = Sigmoid.new
  @reset_sigmoid = Sigmoid.new
  @tanh = Tanh.new
end

Instance Method Details

#backward(dh2) ⇒ Object 



383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
 
# File 'lib/dnn/core/rnn_layers.rb', line 383

def backward(dh2)
  dtanh_h = @tanh.backward(dh2 * @update)
  dh = dh2 * (1 - @update)

  dweight_h = @x.transpose.dot(dtanh_h)
  dx = dtanh_h.dot(@weight_h.transpose)
  dweight2_h = (@h * @reset).transpose.dot(dtanh_h)
  dh += dtanh_h.dot(@weight2_h.transpose) * @reset
  dbias_h = dtanh_h.sum(0)

  dreset = @reset_sigmoid.backward(dtanh_h.dot(@weight2_h.transpose) * @h)
  dupdate = @update_sigmoid.backward(dh2 * @tanh_h - dh2 * @h)
  da = Xumo::SFloat.hstack([dupdate, dreset])
  dweight_a = @x.transpose.dot(da)
  dx += da.dot(@weight_a.transpose)
  dweight2_a = @h.transpose.dot(da)
  dh += da.dot(@weight2_a.transpose)
  dbias_a = da.sum(0)

  @rnn.weight.grad += Xumo::SFloat.hstack([dweight_a, dweight_h])
  @rnn.weight2.grad += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
  if @rnn.l1_lambda > 0
    @rnn.weight.grad += dlasso
    @rnn.weight2.grad += dlasso2
  elsif @rnn.l2_lambda > 0
    @rnn.weight.grad += dridge
    @rnn.weight2.grad += dridge2
  end
  @rnn.bias.grad += Xumo::SFloat.hstack([dbias_a, dbias_h])
  [dx, dh]
end

#forward(x, h) ⇒ Object 



364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
 
# File 'lib/dnn/core/rnn_layers.rb', line 364

def forward(x, h)
  @x = x
  @h = h
  num_nodes = h.shape[1]
  @weight_a = @rnn.weight.data[true, 0...(num_nodes * 2)]
  @weight2_a = @rnn.weight2.data[true, 0...(num_nodes * 2)]
  bias_a = @rnn.bias.data[0...(num_nodes * 2)]
  a = x.dot(@weight_a) + h.dot(@weight2_a) + bias_a
  @update = @update_sigmoid.forward(a[true, 0...num_nodes])
  @reset = @reset_sigmoid.forward(a[true, num_nodes..-1])

  @weight_h = @rnn.weight.data[true, (num_nodes * 2)..-1]
  @weight2_h = @rnn.weight2.data[true, (num_nodes * 2)..-1]
  bias_h = @rnn.bias.data[(num_nodes * 2)..-1]
  @tanh_h = @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
  h2 = (1 - @update) * h + @update * @tanh_h
  h2
end