Module: Stamina::Scoring

Defined in:
lib/stamina-induction/stamina/scoring.rb

Overview

Provides utility methods for scoring binary classifiers from signatures

Constant Summary collapse

MEASURES = 
[
  :false_positive, :false_negative,
  :true_positive, :true_negative,
  :accuracy, :error_rate,
  :precision, :recall, :f_measure,
  :false_positive_rate, :false_negative_rate,
  :true_positive_rate, :true_negative_rate,
  :positive_predictive_value, :negative_predictive_value,
  :sensitivity, :specificity,
  :positive_likelihood, :negative_likelihood,
  :balanced_classification_rate, :balanced_error_rate, :harmonic_bcr
]

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.scoring(learned, actual, max_size = nil) ⇒ Object

From the signatures of a learned model and a actual, returns an object responding to all instance methods defined in the Scoring module.



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 11

def self.scoring(learned, actual, max_size=nil)
  unless learned.size==actual.size
    raise ArgumentError, "Signatures must be of same size (#{learned.size} vs. #{actual.size})"
  end
  max_size ||= learned.size
  max_size = learned.size if max_size > learned.size
  tp, fn, fp, tn = 0, 0, 0, 0
  (0...max_size).each do |i|
    positive, labeled_as = actual[i..i]=='1', learned[i..i]=='1'
    if positive==labeled_as
      positive ? (tp += 1) : (tn += 1)
    else
      positive ? (fn += 1) : (fp += 1)
    end
  end
  measures = { :true_positive  => tp,
               :true_negative  => tn,
               :false_positive => fp,
               :false_negative => fn }
  measures.extend(Scoring)
  measures
end

Instance Method Details

#accuracy ⇒ Object

Returns the percentage of predictions that are correct



127
128
129
130
131
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 127

def accuracy
  num = (true_positive + true_negative).to_f
  den = (true_positive + true_negative + false_positive + false_negative)
  num / den
end

#balanced_classification_rate ⇒ Object Also known as: bcr

Returns the balanced classification rate (arithmetic mean between sensitivity and specificity)



153
154
155
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 153

def balanced_classification_rate
  0.5 * (sensitivity + specificity)
end

#balanced_error_rate ⇒ Object Also known as: ber

Returns the balanced error rate (1 - bcr)



161
162
163
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 161

def balanced_error_rate
  1.0 - balanced_classification_rate
end

#error_rate ⇒ Object

Returns the error rate



136
137
138
139
140
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 136

def error_rate
  num = (false_positive + false_negative).to_f
  den = (true_positive + true_negative + false_positive + false_negative)
  num / den
end

#f_measure ⇒ Object

Returns the harmonic mean between precision and recall



145
146
147
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 145

def f_measure
  2.0 * (precision * recall) / (precision + recall)
end

#false_negative ⇒ Object

Returns the number of positive strings incorrectly labeled as negative.



58
59
60
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 58

def false_negative
  self[:false_negative]
end

#false_negative_rate ⇒ Object

Returns the percentage of false negatives



106
107
108
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 106

def false_negative_rate
  false_negative.to_f / (true_positive + false_negative)
end

#false_positive ⇒ Object

Returns the number of negative strings incorrectly labeled as positive.



51
52
53
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 51

def false_positive
  self[:false_positive]
end

#false_positive_rate ⇒ Object

Returns the percentage of false positives



99
100
101
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 99

def false_positive_rate
  false_positive.to_f / (false_positive + true_negative)
end

#harmonic_balanced_classification_rate ⇒ Object Also known as: hbcr, harmonic_bcr

Returns the harmonic mean between sensitivity and specificity



169
170
171
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 169

def harmonic_balanced_classification_rate
  2.0 * (sensitivity * specificity) / (sensitivity + specificity)
end

#negative_likelihood ⇒ Object

Returns the likelihood that a predicted negative is an actual negative



120
121
122
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 120

def negative_likelihood
 (1.0 - sensitivity) / specificity
end

#negative_predictive_value ⇒ Object

Returns the percentage of true negative over all negative



73
74
75
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 73

def negative_predictive_value
  true_negative.to_f / (true_negative + false_negative)
end

#positive_likelihood ⇒ Object

Returns the likelihood that a predicted positive is an actual positive



113
114
115
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 113

def positive_likelihood
  sensitivity / (1.0 - specificity)
end

#precision ⇒ Object Also known as: positive_predictive_value

Returns the percentage of positive predictions that are correct



65
66
67
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 65

def precision
  true_positive.to_f/(true_positive + false_positive)
end

#recall ⇒ Object Also known as: sensitivity, true_positive_rate

Returns the percentage of positive strings that were predicted as being positive



81
82
83
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 81

def recall
  true_positive.to_f / (true_positive + false_negative)
end

#specificity ⇒ Object Also known as: true_negative_rate

Returns the percentage of negative strings that were predicted as being negative



91
92
93
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 91

def specificity
  true_negative.to_f / (true_negative + false_positive)
end

#to_h ⇒ Object 



188
189
190
191
192
193
194
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 188

def to_h
  h = {}
  MEASURES.each do |m|
    h[m] = self.send(m.to_sym)
  end
  h
end

#to_s ⇒ Object 



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 196

def to_s
  s = ""
  MEASURES.each do |m|
    vals = case val = self.send(m.to_sym)
      when Integer
        "%s" % val
      when Float
        "%.5f" % val
      else
        "%s" % val
    end
    s += "%30s: %10s\n" % [m.to_s, vals]
  end
  s
end

#true_negative ⇒ Object

Returns the number of negative strings correctly labeled as negative.



44
45
46
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 44

def true_negative
  self[:true_negative]
end

#true_positive ⇒ Object

Returns the number of positive strings correctly labeled as positive



37
38
39
 
# File 'lib/stamina-induction/stamina/scoring.rb', line 37

def true_positive
  self[:true_positive]
end