Class: DNN::Layers::GRU_Dense

Inherits:
Object
  • Object
show all
Defined in:
lib/dnn/core/rnn_layers.rb

Instance Method Summary collapse

Constructor Details

#initialize(params, grads) ⇒ GRU_Dense

Returns a new instance of GRU_Dense.



303
304
305
306
307
308
309
 
# File 'lib/dnn/core/rnn_layers.rb', line 303

def initialize(params, grads)
  @params = params
  @grads = grads
  @update_sigmoid = Sigmoid.new
  @reset_sigmoid = Sigmoid.new
  @tanh = Tanh.new
end

Instance Method Details

#backward(dh2) ⇒ Object 



330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/dnn/core/rnn_layers.rb', line 330

def backward(dh2)
  dtanh_h = @tanh.backward(dh2 * @update)
  dh = dh2 * (1 - @update)

  dweight_h = @x.transpose.dot(dtanh_h)
  dx = dtanh_h.dot(@weight_h.transpose)
  dweight2_h = (@h * @reset).transpose.dot(dtanh_h)
  dh += dtanh_h.dot(@weight2_h.transpose) * @reset
  dbias_h = dtanh_h.sum(0)

  dreset = @reset_sigmoid.backward(dtanh_h.dot(@weight2_h.transpose) * @h)
  dupdate = @update_sigmoid.backward(dh2 * @tanh_h - dh2 * @h)
  da = Xumo::SFloat.hstack([dupdate, dreset])
  dweight_a = @x.transpose.dot(da)
  dx += da.dot(@weight_a.transpose)
  dweight2_a = @h.transpose.dot(da)
  dh += da.dot(@weight2_a.transpose)
  dbias_a = da.sum(0)

  @grads[:weight] += Xumo::SFloat.hstack([dweight_a, dweight_h])
  @grads[:weight2] += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
  @grads[:bias] += Xumo::SFloat.hstack([dbias_a, dbias_h])
  [dx, dh]
end

#forward(x, h) ⇒ Object 



311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/dnn/core/rnn_layers.rb', line 311

def forward(x, h)
  @x = x
  @h = h
  num_nodes = h.shape[1]
  @weight_a = @params[:weight][true, 0...(num_nodes * 2)]
  @weight2_a = @params[:weight2][true, 0...(num_nodes * 2)]
  bias_a = @params[:bias][0...(num_nodes * 2)]
  a = x.dot(@weight_a) + h.dot(@weight2_a) + bias_a
  @update = @update_sigmoid.forward(a[true, 0...num_nodes])
  @reset = @reset_sigmoid.forward(a[true, num_nodes..-1])

  @weight_h = @params[:weight][true, (num_nodes * 2)..-1]
  @weight2_h = @params[:weight2][true, (num_nodes * 2)..-1]
  bias_h = @params[:bias][(num_nodes * 2)..-1]
  @tanh_h = @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
  h2 = (1 - @update) * h + @update * @tanh_h
  h2
end