Class: AnswerFactory::Machines::EvaluateWithTestCases

Inherits:
Machine
  • Object
show all
Defined in:
lib/machines/evaluate_with_test_cases.rb

Instance Attribute Summary collapse

Attributes inherited from Machine

#options

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ EvaluateWithTestCases

Returns a new instance of EvaluateWithTestCases.



12
13
14
15
16
17
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 12

def initialize(options = {})
  super
  @name = options[:name] || "evaluator"
  @sensors = options[:sensors] || {}
  @csv_filename = options[:training_data_csv]
end

Instance Attribute Details

#csv_filename ⇒ Object (readonly)

Returns the value of attribute csv_filename.



10
11
12
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 10

def csv_filename
  @csv_filename
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



9
10
11
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 9

def name
  @name
end

#sensors ⇒ Object

Returns the value of attribute sensors.



7
8
9
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 7

def sensors
  @sensors
end

#test_cases ⇒ Object

Returns the value of attribute test_cases.



8
9
10
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 8

def test_cases
  @test_cases
end

Instance Method Details

#build_sensor(name, &block) ⇒ Object 



83
84
85
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 83

def build_sensor(name, &block)
  @sensors[name] = block
end

#csv_headers(csv_reader) ⇒ Object 



118
119
120
121
122
123
124
125
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 118

def csv_headers(csv_reader)
  csv_reader.readline
  split_point = csv_reader.headers.find_index(nil)
  input_headers = csv_reader.headers[0...split_point].collect {|head| header_prep(head)}
  output_headers = csv_reader.headers[split_point+1..-1].collect {|head| head.strip}
  csv_reader.rewind
  return {input_headers:input_headers, output_headers:output_headers}
end

#header_prep(header_string) ⇒ Object

Raises:

  • (ArgumentError) 


76
77
78
79
80
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 76

def header_prep(header_string)
  raise ArgumentError, "Header must match /reference_name:nudge_type/" unless
    header_string.match /[\p{Alpha}][\p{Alnum}_]*:[\p{Alpha}][\p{Alnum}_]/
  header_string.strip
end

#install_training_data_from_csv(csv_filename = @csv_filename) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 95

def install_training_data_from_csv(csv_filename = @csv_filename)
  
  reader = CSV.new(File.open(csv_filename), headers: true)
  headers = csv_headers(reader)
  
  save_view_doc!
  
  offset = headers[:input_headers].length+1
  db = CouchRest.database!(training_datasource)
  
  reader.each do |row|
    inputs = {}
    headers[:input_headers].each_with_index {|header,i| inputs[header] = row[i].strip}
    outputs = {}
    headers[:output_headers].each_with_index {|header,i| outputs[header] = row[i+offset].strip}
    db.bulk_save_doc( {:inputs => inputs, :outputs => outputs})
  end
  
  db.bulk_save
  
end

#load_training_data! ⇒ Object 



128
129
130
131
132
133
134
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 128

def load_training_data!
  db = CouchRest.database!(training_datasource)
  result = db.view("#{@name}/test_cases")
  @test_cases = 
    result["rows"].collect {|r| TestCase.new(
        inputs: r["value"]["inputs"], outputs: r["value"]["outputs"])}
end

#save_view_doc! ⇒ Object 



88
89
90
91
92
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 88

def save_view_doc!
  db = CouchRest.database!(training_datasource)
  db.save_doc({'_id' => "_design/#{@name}",
    views: { test_cases: { map: "function(doc) { emit(doc._id, doc); }"}}})
end

#score(batch, overridden_options = {}) ⇒ Object Also known as: generate

Raises:

  • (ArgumentError) 


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 20

def score(batch, overridden_options = {})
  all_options = @options.merge(overridden_options)
  
  score_name = all_options[:name]
  
  raise ArgumentError, "EvaluateWithTestCases#score cannot process a #{batch.class}" unless
    batch.kind_of?(Batch)
  raise ArgumentError, "EvaluateWithTestCases: Undefined #name attribute" if
    name.nil?
  
  load_training_data!
  
  batch.each do |answer|
    
    
    raw_results = Hash.new {|hash, key| hash[key] = []}
    
    @test_cases.each do |t|
      
      interpreter = Interpreter.new(answer.blueprint,all_options)
      
      t.inputs.each do |variable_header, variable_value|
        variable_name, variable_type = variable_header.split(":")
        interpreter.bind_variable(variable_name, ValuePoint.new(variable_type, variable_value))
      end
      
      @sensors.each do |sensor_name, sensor_block|
        interpreter.register_sensor(sensor_name, &sensor_block)
      end
      
      interpreter.run.each do |sensor_name, sensor_result|
        raw_results[sensor_name] << (t.outputs[sensor_name].to_i - sensor_result)
      end
    end
    
    @sensors.each do |sensor_name, sensor_block|
      answer.scores["#{score_name}_#{sensor_name}".to_sym] =
        raw_results[sensor_name].inject(0) do |sum, measurement|
          sum + measurement.abs
        end
    end
  end
  return batch
end

#training_data_view ⇒ Object 



71
72
73
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 71

def training_data_view
 "#{configatron.factory.training_datasource}/_design/#{@name}/_view/test_cases"
end

#training_datasource ⇒ Object 



66
67
68
 
# File 'lib/machines/evaluate_with_test_cases.rb', line 66

def training_datasource
  configatron.factory.training_datasource
end