Class: Experiment::Base

Inherits:
Object
  • Object
show all
Defined in:
lib/experiment/base.rb

Overview

The base class for defining experimental conditons.

See Also:

Author:

  • Jakub Hampl

Direct Known Subclasses

Factorial

Constant Summary collapse

@@cleanup_raw_files = 
false

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(mode, experiment, options) ⇒ Base

Called internally by the framewrok

Parameters:

  • mode (:normal, :master, :slave)
  • experiment (String) —

    Name of the experimental condition.

  • options (OpenStruct) —

    Most of the options passed from the CLI.



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/experiment/base.rb', line 34

def initialize(mode, experiment, options)
	@experiment = experiment
	@options = options
	# a bit of dependency injection here
	case mode
	when :normal
	  @abm = [] 
 when :master
   @abm = []
   extend DRb::DRbUndumped
   extend Distributed::Master
   @done = false
  when :slave
   extend Distributed::Slave
	end
	
	Experiment::Config::load(experiment, options.opts, options.env)
	@mode = mode
end

Instance Attribute Details

#current_cv ⇒ Object (readonly)

The number of the current cross-validation



22
23
24
 
# File 'lib/experiment/base.rb', line 22

def current_cv
  @current_cv
end

#cvs ⇒ Object (readonly)

The number of overall cross-validations



24
25
26
 
# File 'lib/experiment/base.rb', line 24

def cvs
  @cvs
end

#dir ⇒ Object (readonly)

The directory in which the results will be written to.



20
21
22
 
# File 'lib/experiment/base.rb', line 20

def dir
  @dir
end

#output_file ⇒ Object (readonly)

The file the program is currently set to output to. Use this if you want to write additional data.



27
28
29
 
# File 'lib/experiment/base.rb', line 27

def output_file
  @output_file
end

Class Method Details

.after_completion(*args) ⇒ Object

Sets up actions to do after the task is completed.

This will be expanded in the future. Currently the only possible usage is with :delete_raw_files

Examples:

after_completion :delete_raw_files

Parameters:

  • args (:delete_raw_files) —

    If called will delete the raw-*.txt files in the #dir after the experiment successfully completes.



77
78
79
 
# File 'lib/experiment/base.rb', line 77

def self.after_completion(*args)
  @@cleanup_raw_files = args.include? :delete_raw_files
end

Instance Method Details

#analyze_result!(input, output) ⇒ Hash

This method is abstract.

Override for your own method analysis.

The default analysis function Not terribly useful, better to override

Parameters:

  • input (String) —

    file path of results written by ‘measure` calls.

  • output (String) —

    file path where to optionally write detailed analysis.

Returns:

  • (Hash) —

    Summary of analysis.



65
66
67
 
# File 'lib/experiment/base.rb', line 65

def analyze_result!(input, output)
  YAML::load_file(input)
end

#benchmark(label = "", &block) ⇒ Object

Registers and performs a benchmark which is then calculated to the total and everage times.

A lower-level alternative to measure.



146
147
148
149
 
# File 'lib/experiment/base.rb', line 146

def benchmark(label = "", &block)
  @bm ||= []
  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
end

#data_set ⇒ Object

This method is abstract.

Override this method to return an array.

A silly method meant to be overriden. should return an array, which will be then split up for cross-validating.



193
194
195
 
# File 'lib/experiment/base.rb', line 193

def data_set
  (1..cvs).to_a
end

#done? ⇒ Boolean

Is the experiment done.

Returns:

  • (Boolean) 


55
56
57
 
# File 'lib/experiment/base.rb', line 55

def done?
  @done
end

#measure(label = "", weight = nil, &block) ⇒ Object

Use this every time you want to do a measurement. It will be put on the record file and benchmarked automatically.

Parameters:

  • weight (Integer) (defaults to: nil) —

    Used for calculating Notify::step. It should be an integer denoting how many such measurements you wish to do.



128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/experiment/base.rb', line 128

def measure(label = "", weight = nil, &block)
  out = ""
  benchmark label do
    out = yield
  end
  if out.is_a? String
    @ouptut_file << out
  else
    YAML::dump(out, @ouptut_file)
  end
  Notify::step(@experiment, @current_cv, 1.0/weight) unless weight.nil?
end

#run!(cv) ⇒ Object

runs the whole experiment, called by the framework



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/experiment/base.rb', line 83

def run!(cv)
	@cvs = cv || 1
   @results = {}
	Notify.started @experiment
   split_up_data
	write_dir!
	@cvs.times do |cv_num|
		@bm = []
		@current_cv = cv_num
		File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
		  @ouptut_file = output
		    run_the_experiment
		end
		array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
		write_performance!
		Notify.cv_done @experiment, cv_num
	end
	summarize_performance!
	summarize_results! @results
	specification!
	cleanup!
	Notify.completed @experiment
	puts File.read(@dir + "/summary.mmd") if @options.summary
end

#summarize_results!(results) ⇒ Object

creates a summary of the results and writes to ‘summary.mmd’



155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/experiment/base.rb', line 155

def summarize_results!(results)
  File.open(@dir + '/results.yaml', 'w' ) do |out|
		YAML.dump(results, out)
	end
	# create an array of arrays
	res = results.keys.map do |key| 
	  # calculate stats
	  a = results[key]
	  if a.all? {|el| el.is_a? Numeric }
	    [key] + a + [Stats::mean(a), Stats::standard_deviation(a)]
   else
     [key] + a + ["--", "--"]
    end
 end
 
 ls = results.keys.map{|v| [7, v.to_s.length].max }
	
	ls = ["Std Deviation".length] + ls
	res = header_column + res
	res = res.transpose
	out = build_table res, ls
	File.open(@dir + "/summary.mmd", 'w') do |f|
	  f << "## Results for #{@experiment} ##\n\n"
	  f << out
 end
  #results = results.reduce({}) do |tot, res|
  # cv = res.delete :cv
  # tot.merge Hash[res.to_a.map {|a| ["cv_#{cv}_#{a.first}".to_sym, a.last]}]
  #end
  #FasterCSV.open("./results/all.csv", "a") do |csv|
  #  csv << results.to_a.sort_by{|a| a.first.to_s}.map(&:last)
  #end
end

#test_data ⇒ Array

Returns the portion of the #data_set that corresponds to the current cross validation number.

Returns:

  • (Array) 


111
112
113
 
# File 'lib/experiment/base.rb', line 111

def test_data
  @data[@current_cv]
end

#training_data ⇒ Array

Returns the #data_set that without the #test_data.

Returns:

  • (Array) 


117
118
119
 
# File 'lib/experiment/base.rb', line 117

def training_data
  (@data - test_data).flatten
end