Class: NluAdapter::Metrics

Inherits:
Object
  • Object
show all
Defined in:
lib/nlu_adapter/metrics.rb

Overview

A utility class to calculate classification matrics

See Also:

Instance Method Summary collapse

Constructor Details

#initialize(actual, predicted, class_labels = []) ⇒ Metrics

Constructor



16
17
18
19
20
21
22
23
24
 
# File 'lib/nlu_adapter/metrics.rb', line 16

def initialize(actual, predicted, class_labels = [])
  @actual = actual
  @predicted = predicted
  @class_labels = class_labels

  @actual.map!{ |a| a.is_a?(String)? a.intern : a }
  @predicted.map!{ |p| p.is_a?(String)? p.intern : p }
  @class_labels.map!{ |l| l.is_a?(String)? l.intern : l }
end

Instance Method Details

#accuracy ⇒ Float

Caclulate the accuracy

See Also:



30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/nlu_adapter/metrics.rb', line 30

def accuracy
  if @actual.size != @predicted.size
    #todo: throw error
    puts "actual & predicted array are not of same size"
    return
  end

  total = @actual.size
  correct = 0
  @actual.each_with_index do |v, i|
    correct+=1 if @predicted[i] == v
  end
  (correct.fdiv(total) * 100).round(4)
end

#class_totals ⇒ Hash

Get totals of actual values per class



98
99
100
 
# File 'lib/nlu_adapter/metrics.rb', line 98

def class_totals
  return @class_totals
end

#classification_report ⇒ Hash

Generate classification report



161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/nlu_adapter/metrics.rb', line 161

def classification_report
  confusion_matrix if !@m
  report = {}
  @class_labels.each do |label|
    report[label] = {
      precision: precision(label),
      recall: recall(label),
      class_total: class_totals[label]
    }
  end

  return report
end

#confusion_matrix ⇒ Matrix

Get the confusion matrix

See Also:



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/nlu_adapter/metrics.rb', line 49

def confusion_matrix
  class_labels = @class_labels
  actual = @actual
  predicted = @predicted

  #if no class_labels, convert to numeric values, and extract the class_labels
  if @class_labels.empty?
    class_labels = (@actual + @predicted).sort.uniq.sort
    class_labels.map!{|c| c.intern}
    @actual.each_with_index do |v, i|
      actual[i] = class_labels.index(v)
    end

    @predicted.each_with_index do |v, i|
      predicted[i] = class_labels.index(v)
    end
  else
    #check if any string passed in actual/predicted
    i = actual.select { |a| a.is_a?(Symbol) }.size
    j = predicted.select { |p| p.is_a?(Symbol) }.size

    if i > 0 || j > 0
      #todo: fix it OR throw error
      puts "actual, predicted & class_labels array having string values not implemented yet"
      return

    end
  end

  m = Matrix.zero(class_labels.size)

  @class_totals = Hash[class_labels.collect { |c| [c, 0] }]
  actual.each_with_index do |vi, i|
    vj = predicted[i]
    m[vi, vj] = m[vi, vj] + 1
    @class_totals[class_labels[vi]] += 1
  end

  @class_labels = class_labels
  @actual = actual
  @predicted = predicted

  @m = m
  return m
end

#fn(class_name) ⇒ Integer

Get false negative



128
129
130
131
132
133
 
# File 'lib/nlu_adapter/metrics.rb', line 128

def fn(class_name)
  i = @class_labels.index(class_name.intern)
  return nil if i == nil || @m == nil || @m.empty?
  fn = @m.row(i).sum - tp(class_name)
  return fn
end

#fp(class_name) ⇒ Integer

Get false positive



117
118
119
120
121
122
 
# File 'lib/nlu_adapter/metrics.rb', line 117

def fp(class_name)
  i = @class_labels.index(class_name.intern)
  return nil if i == nil || @m == nil || @m.empty?
  fp = @m.column(i).sum - tp(class_name)
  return fp
end

#precision(class_name) ⇒ Float

Get the precision for given class



139
140
141
142
143
144
 
# File 'lib/nlu_adapter/metrics.rb', line 139

def precision(class_name)
  confusion_matrix if !@m
  return 0.00 if tp(class_name) == 0
  precision = tp(class_name).fdiv((tp(class_name) + fp(class_name))).round(4)
  return precision
end

#recall(class_name) ⇒ Float

Get the recall for given class



150
151
152
153
154
155
156
 
# File 'lib/nlu_adapter/metrics.rb', line 150

def recall(class_name)
  confusion_matrix if !@m
  return 0.00 if tp(class_name) == 0
  recall = tp(class_name).fdiv((tp(class_name) + fn(class_name))).round(4)

  return recall
end

#tp(class_name) ⇒ Integer

Get total positives



106
107
108
109
110
111
 
# File 'lib/nlu_adapter/metrics.rb', line 106

def tp(class_name)
  i = @class_labels.index(class_name.intern)
  return nil if i == nil || @m == nil || @m.empty?
  tp = @m[i, i]
  return tp
end