Class: DNN::Layers::RNN

Inherits:
Connection show all
Includes:
Activations
Defined in:
lib/dnn/core/rnn_layers.rb

Overview

Super class of all RNN classes.

Direct Known Subclasses

GRU, LSTM, SimpleRNN

Instance Attribute Summary collapse

Attributes inherited from Connection

#bias_initializer, #l1_lambda, #l2_lambda, #weight_initializer

Attributes inherited from HasParamLayer

#params, #trainable

Attributes inherited from Layer

#input_shape

Instance Method Summary collapse

Methods inherited from HasParamLayer

#build, #update

Methods inherited from Layer

#build, #built?

Constructor Details

#initialize(num_nodes, stateful: false, return_sequences: true, weight_initializer: Initializers::RandomNormal.new, bias_initializer: Initializers::Zeros.new, l1_lambda: 0, l2_lambda: 0) ⇒ RNN

Returns a new instance of RNN.



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/dnn/core/rnn_layers.rb', line 14

def initialize(num_nodes,
               stateful: false,
               return_sequences: true,
               weight_initializer: Initializers::RandomNormal.new,
               bias_initializer: Initializers::Zeros.new,
               l1_lambda: 0,
               l2_lambda: 0)
  super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
        l1_lambda: l1_lambda, l2_lambda: l2_lambda)
  @num_nodes = num_nodes
  @stateful = stateful
  @return_sequences = return_sequences
  @layers = []
  @hidden = @params[:h] = Param.new
  @params[:weight2] = @weight2 = Param.new
end

Instance Attribute Details

#bias ⇒ Object (readonly)

Returns the value of attribute bias.



12
13
14
 
# File 'lib/dnn/core/rnn_layers.rb', line 12

def bias
  @bias
end

#num_nodes ⇒ Object (readonly)

Returns the value of attribute num_nodes.



8
9
10
 
# File 'lib/dnn/core/rnn_layers.rb', line 8

def num_nodes
  @num_nodes
end

#stateful ⇒ Object (readonly)

Returns the value of attribute stateful.



9
10
11
 
# File 'lib/dnn/core/rnn_layers.rb', line 9

def stateful
  @stateful
end

#weight ⇒ Object (readonly)

Returns the value of attribute weight.



10
11
12
 
# File 'lib/dnn/core/rnn_layers.rb', line 10

def weight
  @weight
end

#weight2 ⇒ Object (readonly)

Returns the value of attribute weight2.



11
12
13
 
# File 'lib/dnn/core/rnn_layers.rb', line 11

def weight2
  @weight2
end

Instance Method Details

#backward(dh2s) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/dnn/core/rnn_layers.rb', line 44

def backward(dh2s)
  @weight.grad = Xumo::SFloat.zeros(*@weight.data.shape)
  @weight2.grad = Xumo::SFloat.zeros(*@weight2.data.shape)
  @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape)
  unless @return_sequences
    dh = dh2s
    dh2s = Xumo::SFloat.zeros(dh.shape[0], @time_length, dh.shape[1])
    dh2s[true, -1, false] = dh
  end
  dxs = Xumo::SFloat.zeros(@xs_shape)
  dh = 0
  (0...dh2s.shape[1]).to_a.reverse.each do |t|
    dh2 = dh2s[true, t, false]
    dx, dh = @layers[t].backward(dh2 + dh)
    dxs[true, t, false] = dx
  end
  dxs
end

#d_lasso ⇒ Object 



101
102
103
104
105
106
107
108
109
110
 
# File 'lib/dnn/core/rnn_layers.rb', line 101

def d_lasso
  if @l1_lambda > 0
    dlasso = Xumo::SFloat.ones(*@weight.data.shape)
    dlasso[@weight.data < 0] = -1
    @weight.grad += @l1_lambda * dlasso
    dlasso2 = Xumo::SFloat.ones(*@weight2.data.shape)
    dlasso2[@weight2.data < 0] = -1
    @weight2.grad += @l1_lambda * dlasso2
  end
end

#d_ridge ⇒ Object 



112
113
114
115
116
117
 
# File 'lib/dnn/core/rnn_layers.rb', line 112

def d_ridge
  if @l2_lambda > 0
    @weight.grad += l2_lambda * @weight.data
    @weight2.grad += l2_lambda * @weight2.data
  end
end

#forward(xs) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/dnn/core/rnn_layers.rb', line 31

def forward(xs)
  @xs_shape = xs.shape
  hs = Xumo::SFloat.zeros(xs.shape[0], @time_length, @num_nodes)
  h = (@stateful && @hidden.data) ? @hidden.data : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
  xs.shape[1].times do |t|
    x = xs[true, t, false]
    h = @layers[t].forward(x, h)
    hs[true, t, false] = h
  end
  @hidden.data = h
  @return_sequences ? hs : h
end

#lasso ⇒ Object 



85
86
87
88
89
90
91
 
# File 'lib/dnn/core/rnn_layers.rb', line 85

def lasso
  if @l1_lambda > 0
    @l1_lambda * (@weight.data.abs.sum + @weight2.data.abs.sum)
  else
    0
  end
end

#output_shape ⇒ Object 



63
64
65
 
# File 'lib/dnn/core/rnn_layers.rb', line 63

def output_shape
  @return_sequences ? [@time_length, @num_nodes] : [@num_nodes]
end

#reset_state ⇒ Object 



81
82
83
 
# File 'lib/dnn/core/rnn_layers.rb', line 81

def reset_state
  @hidden.data = @hidden.data.fill(0) if @hidden.data
end

#ridge ⇒ Object 



93
94
95
96
97
98
99
 
# File 'lib/dnn/core/rnn_layers.rb', line 93

def ridge
  if @l2_lambda > 0
    0.5 * (@l2_lambda * ((@weight.data**2).sum + (@weight2.data**2).sum))
  else
    0
  end
end

#shape ⇒ Object 



77
78
79
 
# File 'lib/dnn/core/rnn_layers.rb', line 77

def shape
  @return_sequences ? [@time_length, @num_nodes] : [@num_nodes]
end

#to_hash(merge_hash = nil) ⇒ Object 



67
68
69
70
71
72
73
74
75
 
# File 'lib/dnn/core/rnn_layers.rb', line 67

def to_hash(merge_hash = nil)
  hash = {
    num_nodes: @num_nodes,
    stateful: @stateful,
    return_sequences: @return_sequences
  }
  hash.merge!(merge_hash) if merge_hash
  super(hash)
end