Class: Experiment::Base

Inherits:
Object
  • Object
show all
Defined in:
lib/experiment/base.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(experiment, options, env) ⇒ Base

Returns a new instance of Base.



10
11
12
13
14
15
16
 
# File 'lib/experiment/base.rb', line 10

def initialize(experiment, options, env)
	@experiment = experiment
	
	Experiment::Config::load(experiment, options, env)
	require "./experiments/#{experiment}/#{experiment}"
	@abm = []
end

Instance Attribute Details

#current_cv ⇒ Object (readonly)

Returns the value of attribute current_cv.



8
9
10
 
# File 'lib/experiment/base.rb', line 8

def current_cv
  @current_cv
end

#cvs ⇒ Object (readonly)

Returns the value of attribute cvs.



8
9
10
 
# File 'lib/experiment/base.rb', line 8

def cvs
  @cvs
end

#dir ⇒ Object (readonly)

Returns the value of attribute dir.



8
9
10
 
# File 'lib/experiment/base.rb', line 8

def dir
  @dir
end

Instance Method Details

#benchmark(label = "", &block) ⇒ Object

Registers and performs a benchmark which is then calculated to the total and everage times



45
46
47
 
# File 'lib/experiment/base.rb', line 45

def benchmark(label = "", &block)
  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
end

#result_line ⇒ Object 



130
131
132
 
# File 'lib/experiment/base.rb', line 130

def result_line
 " Done\n"
end

#run!(cv) ⇒ Object

runs the whole experiment



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/experiment/base.rb', line 19

def run!(cv)
	@cvs = cv || 1
   @results = {}
	Notify.print "Running #{@experiment} "
   split_up_data
	write_dir!
	specification!

	@cvs.times do |cv_num|
		@bm = []
		@current_cv = cv_num
		File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
		    run_the_experiment(@data[cv_num], output)
		end
		array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
		write_performance!
		Notify.print "."
	end
	summarize_performance!
	summarize_results! @results
	Notify.print result_line
end

#specification! ⇒ Object

Writes a yaml specification of all the options used to run the experiment



57
58
59
60
61
 
# File 'lib/experiment/base.rb', line 57

def specification!
	File.open(@dir + '/specification.yaml', 'w' ) do |out|
		YAML.dump({:name => @experiment, :date => Time.now, :configuration => Experiment::Config.to_h, :cross_validations => @cvs}, out )
	end
end

#split_up_data ⇒ Object 



140
141
142
143
144
145
146
147
 
# File 'lib/experiment/base.rb', line 140

def split_up_data
  @data = []
  test_data.each_with_index do |item, i|
    @data[i % cvs] ||= []
    @data[i % cvs] << item
  end
  @data
end

#summarize_performance! ⇒ Object

Calculates the average performance and writes it



78
79
80
81
82
83
 
# File 'lib/experiment/base.rb', line 78

def summarize_performance!
	performance_f do |f|
		total = @abm.reduce(0) {|t, m| m + t} / @abm.count
		f << total.format("       Average: "+Benchmark::FMTSTR)
	end
end

#summarize_results!(results) ⇒ Object

creates a summary of the results and writes to ‘all.csv’



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/experiment/base.rb', line 86

def summarize_results!(results)
  File.open(@dir + '/results.yaml', 'w' ) do |out|
		YAML.dump(results, out )
	end
	
	# create an array of arrays
	res = results.keys.map do |key| 
	  # calculate stats
	  a = results[key]
	  [key] + a + [Stats::mean(a), Stats::standard_deviation(a)]
 end
 
 ls = results.keys.map{|v| v.to_s.length }
	
	ls = ["Standard Deviation".length] + ls
	res = [["cv"] + (1..cvs).to_a.map(&:to_s) + ["Mean", "Standard Deviation"]] + res
	
	out = ""
	res.transpose.each do |col|
	  col.each_with_index do |cell, i|
	    l = ls[i]
	    out << "| "
	    if cell.is_a?(String) || cell.is_a?(Symbol)
	      out << sprintf("%#{l}s", cell)
     else
       out << sprintf("%#{l}.3f", cell)
     end
     out << " "
	  end
	  out << "|\n"
	end
	File.open(@dir + "/summary.mmd", 'w') do |f|
	  f << "## Results for #{@experiment} ##\n\n"
	  f << out
 end
  #results = results.reduce({}) do |tot, res|
  # cv = res.delete :cv
  # tot.merge Hash[res.to_a.map {|a| ["cv_#{cv}_#{a.first}".to_sym, a.last]}]
  #end
  #FasterCSV.open("./results/all.csv", "a") do |csv|
  #  csv << results.to_a.sort_by{|a| a.first.to_s}.map(&:last)
  #end
end

#test_data ⇒ Object

A silly method meant to be overriden. should return an array, which will be then split up for cross-validating



136
137
138
 
# File 'lib/experiment/base.rb', line 136

def test_data
  (1..cvs).to_a
end

#write_dir! ⇒ Object

Creates the results directory for the current experiment



51
52
53
54
 
# File 'lib/experiment/base.rb', line 51

def write_dir!
	@dir = "./results/#{@experiment}-cv#{@cvs}-#{Time.now.to_i.to_s[4..9]}"
	Dir.mkdir @dir
end

#write_performance! ⇒ Object

Writes a file called ‘performance_table.txt’ which details all the benchmarks performed



67
68
69
70
71
72
73
74
75
 
# File 'lib/experiment/base.rb', line 67

def write_performance!
	performance_f do |f|
		f << "Cross Validation #{@current_cv} " + Benchmark::CAPTION 
		f << @bm.map {|m| m.format("%19n "+Benchmark::FMTSTR)}.join
		total = @bm.reduce(0) {|t, m| m + t}
		f << total.format("         Total: "+Benchmark::FMTSTR)
		@abm << total
	end
end