Class: OpenTox::ClassificationCrossValidation

Inherits:
CrossValidation show all
Defined in:
lib/crossvalidation.rb

Instance Method Summary collapse

Methods inherited from CrossValidation

create, #model, #time, #validations

Instance Method Details

#confidence_plot ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/crossvalidation.rb', line 133

def confidence_plot
  unless confidence_plot_id
    tmpfile = "/tmp/#{id.to_s}_confidence.png"
    accuracies = []
    confidences = []
    correct_predictions = 0
    incorrect_predictions = 0
    predictions.each do |p|
      if p[1] and p[2]
        p[1] == p[2] ? correct_predictions += 1 : incorrect_predictions += 1
        accuracies << correct_predictions/(correct_predictions+incorrect_predictions).to_f
        confidences << p[3]

      end
    end
    R.assign "accuracy", accuracies
    R.assign "confidence", confidences
    R.eval "image = qplot(confidence,accuracy)+ylab('accumulated accuracy')+scale_x_reverse()"
    R.eval "ggsave(file='#{tmpfile}', plot=image)"
    file = Mongo::Grid::File.new(File.read(tmpfile), :filename => "#{self.id.to_s}_confidence_plot.png")
    plot_id = $gridfs.insert_one(file)
    update(:confidence_plot_id => plot_id)
  end
  $gridfs.find_one(_id: confidence_plot_id).data
end

#statistics ⇒ Object

TODO auc, f-measure (usability??)



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/crossvalidation.rb', line 75

def statistics
  accept_values = Feature.find(model.prediction_feature_id).accept_values
  confusion_matrix = Array.new(accept_values.size,0){Array.new(accept_values.size,0)}
  weighted_confusion_matrix = Array.new(accept_values.size,0){Array.new(accept_values.size,0)}
  true_rate = {}
  predictivity = {}
  predictions.each do |pred|
    compound_id,activities,prediction,confidence = pred
    if activities and prediction #and confidence.numeric? 
      if activities.uniq.size == 1
        activity = activities.uniq.first
        if prediction == activity
          if prediction == accept_values[0]
            confusion_matrix[0][0] += 1
            #weighted_confusion_matrix[0][0] += confidence
          elsif prediction == accept_values[1]
            confusion_matrix[1][1] += 1
            #weighted_confusion_matrix[1][1] += confidence
          end
        elsif prediction != activity
          if prediction == accept_values[0]
            confusion_matrix[0][1] += 1
            #weighted_confusion_matrix[0][1] += confidence
          elsif prediction == accept_values[1]
            confusion_matrix[1][0] += 1
            #weighted_confusion_matrix[1][0] += confidence
          end
        end
      end
    else
      nr_unpredicted += 1 if prediction.nil?
    end
  end
  true_rate = {}
  predictivity = {}
  accept_values.each_with_index do |v,i|
    true_rate[v] = confusion_matrix[i][i]/confusion_matrix[i].reduce(:+).to_f
    predictivity[v] = confusion_matrix[i][i]/confusion_matrix.collect{|n| n[i]}.reduce(:+).to_f
  end
  confidence_sum = 0
  #weighted_confusion_matrix.each do |r|
    #r.each do |c|
      #confidence_sum += c
    #end
  #end
  update_attributes(
    accept_values: accept_values,
    confusion_matrix: confusion_matrix,
    #weighted_confusion_matrix: weighted_confusion_matrix,
    accuracy: (confusion_matrix[0][0]+confusion_matrix[1][1])/(nr_instances-nr_unpredicted).to_f,
    #weighted_accuracy: (weighted_confusion_matrix[0][0]+weighted_confusion_matrix[1][1])/confidence_sum.to_f,
    true_rate: true_rate,
    predictivity: predictivity,
    finished_at: Time.now
  )
  $logger.debug "Accuracy #{accuracy}"
end