Class: BackProp::Value

Inherits:
Object
  • Object
show all
Defined in:
lib/backprop.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(float, label: '', op: nil, children: []) ⇒ Value

Returns a new instance of Value.



10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/backprop.rb', line 10

def initialize(float, label: '', op: nil, children: [])
  @value = float.to_f
  @gradient = 0
  @children = children
  if @children.empty?
    raise "op #{op.inspect} has no children" unless op.nil?
  else
    raise "op is required" if op.nil?
  end
  @op = op
  @label = label
  @backstep = -> {}
end

Instance Attribute Details

#backstep ⇒ Object

Returns the value of attribute backstep.



8
9
10
 
# File 'lib/backprop.rb', line 8

def backstep
  @backstep
end

#children ⇒ Object (readonly)

Returns the value of attribute children.



7
8
9
 
# File 'lib/backprop.rb', line 7

def children
  @children
end

#gradient ⇒ Object

Returns the value of attribute gradient.



8
9
10
 
# File 'lib/backprop.rb', line 8

def gradient
  @gradient
end

#label ⇒ Object

Returns the value of attribute label.



8
9
10
 
# File 'lib/backprop.rb', line 8

def label
  @label
end

#op ⇒ Object

Returns the value of attribute op.



8
9
10
 
# File 'lib/backprop.rb', line 8

def op
  @op
end

#value ⇒ Object

Returns the value of attribute value.



8
9
10
 
# File 'lib/backprop.rb', line 8

def value
  @value
end

Class Method Details

.wrap(other) ⇒ Object 



3
4
5
 
# File 'lib/backprop.rb', line 3

def self.wrap(other)
  other.is_a?(Value) ? other : Value.new(other)
end

Instance Method Details

#*(other) ⇒ Object 



57
58
59
60
61
62
63
64
65
 
# File 'lib/backprop.rb', line 57

def *(other)
  other = Value.wrap(other)
  val = Value.new(@value * other.value, children: [self, other], op: :*)
  val.backstep = -> {
    self.gradient += val.gradient * other.value
    other.gradient += val.gradient * self.value
  }
  val
end

#**(other) ⇒ Object

Mostly we are squaring(2) or dividing(-1)



68
69
70
71
72
73
74
75
 
# File 'lib/backprop.rb', line 68

def **(other)
  raise("Value is not supported") if other.is_a? Value
  val = Value.new(@value ** other, children: [self], op: :**)
  val.backstep = -> {
    self.gradient += val.gradient * (other * self.value ** (other - 1))
  }
  val
end

#+(other) ⇒ Object

Primary operations; notice every Value.new(op:) also defines a backstep

The backstep closes over the environment of the method so it can
refer to values present when the method executes



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/backprop.rb', line 46

def +(other)
  other = Value.wrap(other)
  val = Value.new(@value + other.value, children: [self, other], op: :+)
  val.backstep = -> {
    # gradients accumulate to handle a value used multiple times
    self.gradient += val.gradient
    other.gradient += val.gradient
  }
  val
end

#-(other) ⇒ Object

Secondary operations defined in terms of primary



89
90
91
 
# File 'lib/backprop.rb', line 89

def -(other)
  self + (Value.wrap(other) * Value.new(-1))
end

#/(other) ⇒ Object 



93
94
95
 
# File 'lib/backprop.rb', line 93

def /(other)
  self * (Value.wrap(other) ** -1)
end

#backprop ⇒ Object 



140
141
142
143
144
 
# File 'lib/backprop.rb', line 140

def backprop
  self.backstep.call
  @children.each(&:backprop)
  self
end

#backward ⇒ Object

Backward propagation



128
129
130
131
132
 
# File 'lib/backprop.rb', line 128

def backward
  self.reset_gradient
  @gradient = 1.0
  self.backprop
end

#display ⇒ Object 



28
29
30
31
32
33
 
# File 'lib/backprop.rb', line 28

def display
  format("%s(%.3f gradient=%.3f",
         @label.empty? ? @op || 'Value' : @label, @value, @gradient) +
    (@op.nil? ? '' :
       format(" %s(%s)", @op, @children.join(', '))) + ')'
end

#exp ⇒ Object 



77
78
79
80
81
82
83
 
# File 'lib/backprop.rb', line 77

def exp
  val = Value.new(Math.exp(@value), children: [self], op: :exp)
  val.backstep = -> {
    self.gradient += val.gradient * val.value
  }
  val
end

#inspect ⇒ Object 



35
36
37
38
 
# File 'lib/backprop.rb', line 35

def inspect
  @children.empty? ? self.display :
    [self.display, @children.map(&:inspect).join("\n\t")].join("\n\t")
end

#relu ⇒ Object

rectified linear unit; not susceptible to vanishing gradient like above



115
116
117
118
119
120
121
122
 
# File 'lib/backprop.rb', line 115

def relu
  neg = @value < 0
  val = Value.new(neg ? 0 : @value, children: [self], op: :relu)
  val.backstep = -> {
    self.gradient += val.gradient * (neg ? 0 : 1)
  }
  val
end

#reset_gradient ⇒ Object 



134
135
136
137
138
 
# File 'lib/backprop.rb', line 134

def reset_gradient
  @gradient = 0.0
  @children.each(&:reset_gradient)
  self
end

#sigmoid ⇒ Object

1 / 1 + e^-x



110
111
112
 
# File 'lib/backprop.rb', line 110

def sigmoid
  ((self * -1).exp + 1) ** -1
end

#tanh ⇒ Object

Activation functions



101
102
103
104
105
106
107
 
# File 'lib/backprop.rb', line 101

def tanh
  val = Value.new(Math.tanh(@value), children: [self], op: :tanh)
  val.backstep = -> {
    self.gradient += val.gradient * (1 - val.value ** 2)
  }
  val
end

#to_s ⇒ Object 



24
25
26
 
# File 'lib/backprop.rb', line 24

def to_s
  @label.empty? ? ("%.3f" % @value) : format("%s=%.3f", @label, @value)
end