Class: Experiment::Base

Inherits:
Object
  • Object
show all
Includes:
Distributed
Defined in:
lib/experiment/base.rb

Overview

The base class for defining experimental conditons.

See Also:

Author:

  • Jakub Hampl

Direct Known Subclasses

Factorial

Constant Summary collapse

@@cleanup_raw_files = 
false

Instance Attribute Summary collapse

Attributes included from Distributed

#master

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Distributed

#distribution_done?, #get_work, #master_done!, #master_run!, #slave_run!, #submit_result

Constructor Details

#initialize(mode, experiment, options) ⇒ Base

Called internally by the framewrok

Parameters:

  • mode (:normal, :master, :slave)
  • experiment (String) —

    name



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/experiment/base.rb', line 34

def initialize(mode, experiment, options)
	@experiment = experiment
	@options = options
	case mode
	  
	when :normal
	  @abm = [] 
	  
 when :master
   @abm = []
   extend DRb::DRbUndumped
   @done = false
  when :slave
   
	end
	Experiment::Config::load(experiment, options.opts, options.env)
	@mode = mode
end

Instance Attribute Details

#current_cv ⇒ Object (readonly)

The number of the current cross-validation



23
24
25
 
# File 'lib/experiment/base.rb', line 23

def current_cv
  @current_cv
end

#cvs ⇒ Object (readonly)

The number of overall cross-validations



25
26
27
 
# File 'lib/experiment/base.rb', line 25

def cvs
  @cvs
end

#dir ⇒ Object (readonly)

The directory in which the results will be written to.



21
22
23
 
# File 'lib/experiment/base.rb', line 21

def dir
  @dir
end

#output_file ⇒ Object (readonly)

The file the program is currently set to output to. Use this if you want to write additional data.



28
29
30
 
# File 'lib/experiment/base.rb', line 28

def output_file
  @output_file
end

Class Method Details

.after_completion(*args) ⇒ Object

Sets up actions to do after the task is completed.

This will be expanded in the future. Currently the only possible usage is with :delete_raw_files

Examples:

after_completion :delete_raw_files

Parameters:

  • args (:delete_raw_files) —

    If called will delete the raw-*.txt files in the #dir after the experiment successfully completes.



76
77
78
 
# File 'lib/experiment/base.rb', line 76

def self.after_completion(*args)
  @@cleanup_raw_files = args.include? :delete_raw_files
end

Instance Method Details

#analyze_result!(input, output) ⇒ Hash

This method is abstract.

Override for your own method analysis.

The default analysis function Not terribly useful, better to override

Parameters:

  • input (String) —

    file path of results written by ‘measure` calls.

  • output (String) —

    file path where to optionally write detailed analysis.

Returns:

  • (Hash) —

    Summary of analysis.



64
65
66
 
# File 'lib/experiment/base.rb', line 64

def analyze_result!(input, output)
  YAML::load_file(input)
end

#benchmark(label = "", &block) ⇒ Object

Registers and performs a benchmark which is then calculated to the total and everage times.

A lower-level alternative to measure.



145
146
147
148
 
# File 'lib/experiment/base.rb', line 145

def benchmark(label = "", &block)
  @bm ||= []
  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
end

#data_set ⇒ Object

This method is abstract.

Override this method to return an array.

A silly method meant to be overriden. should return an array, which will be then split up for cross-validating.



192
193
194
 
# File 'lib/experiment/base.rb', line 192

def data_set
  (1..cvs).to_a
end

#done? ⇒ Boolean

Is the experiment done.

Returns:

  • (Boolean) 


54
55
56
 
# File 'lib/experiment/base.rb', line 54

def done?
  @done
end

#measure(label = "", weight = nil, &block) ⇒ Object

Use this every time you want to do a measurement. It will be put on the record file and benchmarked automatically.

Parameters:

  • weight (Integer) (defaults to: nil) —

    Used for calculating Notify::step. It should be an integer denoting how many such measurements you wish to do.



127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/experiment/base.rb', line 127

def measure(label = "", weight = nil, &block)
  out = ""
  benchmark label do
    out = yield
  end
  if out.is_a? String
    @ouptut_file << out
  else
    YAML::dump(out, @ouptut_file)
  end
  Notify::step(@experiment, @current_cv, 1.0/weight) unless weight.nil?
end

#normal_run!(cv) ⇒ Object

runs the whole experiment, called by the framework



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/experiment/base.rb', line 82

def normal_run!(cv)
	@cvs = cv || 1
   @results = {}
	Notify.started @experiment
   split_up_data
	write_dir!
	@cvs.times do |cv_num|
		@bm = []
		@current_cv = cv_num
		File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
		  @ouptut_file = output
		    run_the_experiment
		end
		array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
		write_performance!
		Notify.cv_done @experiment, cv_num
	end
	summarize_performance!
	summarize_results! @results
	specification!
	cleanup!
	Notify.completed @experiment
	puts File.read(@dir + "/summary.mmd") if @options.summary
end

#summarize_results!(results) ⇒ Object

creates a summary of the results and writes to ‘summary.mmd’



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/experiment/base.rb', line 154

def summarize_results!(results)
  File.open(@dir + '/results.yaml', 'w' ) do |out|
		YAML.dump(results, out)
	end
	# create an array of arrays
	res = results.keys.map do |key| 
	  # calculate stats
	  a = results[key]
	  if a.all? {|el| el.is_a? Numeric }
	    [key] + a + [Stats::mean(a), Stats::standard_deviation(a)]
   else
     [key] + a + ["--", "--"]
    end
 end
 
 ls = results.keys.map{|v| [7, v.to_s.length].max }
	
	ls = ["Std Deviation".length] + ls
	res = header_column + res
	res = res.transpose
	out = build_table res, ls
	File.open(@dir + "/summary.mmd", 'w') do |f|
	  f << "## Results for #{@experiment} ##\n\n"
	  f << out
 end
  #results = results.reduce({}) do |tot, res|
  # cv = res.delete :cv
  # tot.merge Hash[res.to_a.map {|a| ["cv_#{cv}_#{a.first}".to_sym, a.last]}]
  #end
  #FasterCSV.open("./results/all.csv", "a") do |csv|
  #  csv << results.to_a.sort_by{|a| a.first.to_s}.map(&:last)
  #end
end

#test_data ⇒ Array

Returns the portion of the #data_set that corresponds to the current cross validation number.

Returns:

  • (Array) 


110
111
112
 
# File 'lib/experiment/base.rb', line 110

def test_data
  @data[@current_cv]
end

#training_data ⇒ Array

Returns the #data_set that without the #test_data.

Returns:

  • (Array) 


116
117
118
 
# File 'lib/experiment/base.rb', line 116

def training_data
  (@data - test_data).flatten
end