Class: Experiment::Base

Inherits:
Object
  • Object
show all
Includes:
Distributed
Defined in:
lib/experiment/base.rb

Overview

The base class for defining experimental conditons.

See Also:

Author:

  • Jakub Hampl

Direct Known Subclasses

Factorial

Constant Summary collapse

@@cleanup_raw_files = 
false

Instance Attribute Summary collapse

Attributes included from Distributed

#master

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Distributed

#distribution_done?, #get_work, #master_done!, #master_run!, #slave_run!, #submit_result

Constructor Details

#initialize(mode, experiment, options) ⇒ Base

Called internally by the framewrok

Parameters:

  • mode (:normal, :master, :slave)
  • experiment (String) —

    name



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/experiment/base.rb', line 34

def initialize(mode, experiment, options)
  @experiment = experiment
  @options = options
  case mode
    
  when :normal
    @abm = [] 
    
 when :master
   @abm = []
   extend DRb::DRbUndumped
   @done = false
  when :slave
   
  end
  Experiment::Config::load(experiment, options.opts, options.env)
  @mode = mode
end

Instance Attribute Details

#current_cv ⇒ Object (readonly)

The number of the current cross-validation



23
24
25
 
# File 'lib/experiment/base.rb', line 23

def current_cv
  @current_cv
end

#cvs ⇒ Object (readonly)

The number of overall cross-validations



25
26
27
 
# File 'lib/experiment/base.rb', line 25

def cvs
  @cvs
end

#dir ⇒ Object (readonly)

The directory in which the results will be written to.



21
22
23
 
# File 'lib/experiment/base.rb', line 21

def dir
  @dir
end

#output_file ⇒ Object (readonly)

The file the program is currently set to output to. Use this if you want to write additional data.



28
29
30
 
# File 'lib/experiment/base.rb', line 28

def output_file
  @output_file
end

Class Method Details

.after_completion(*args) ⇒ Object

Sets up actions to do after the task is completed.

This will be expanded in the future. Currently the only possible usage is with :delete_raw_files

Examples:

after_completion :delete_raw_files

Parameters:

  • args (:delete_raw_files) —

    If called will delete the raw-*.txt files in the #dir after the experiment successfully completes.



76
77
78
 
# File 'lib/experiment/base.rb', line 76

def self.after_completion(*args)
  @@cleanup_raw_files = args.include? :delete_raw_files
end

Instance Method Details

#analyze_result!(input, output) ⇒ Hash

This method is abstract.

Override for your own method analysis.

The default analysis function Not terribly useful, better to override

Parameters:

  • input (String) —

    file path of results written by ‘measure` calls.

  • output (String) —

    file path where to optionally write detailed analysis.

Returns:

  • (Hash) —

    Summary of analysis.



64
65
66
 
# File 'lib/experiment/base.rb', line 64

def analyze_result!(input, output)
  YAML::load_file(input)
end

#benchmark(label = "", &block) ⇒ Object

Registers and performs a benchmark which is then calculated to the total and everage times.

A lower-level alternative to measure.



145
146
147
148
 
# File 'lib/experiment/base.rb', line 145

def benchmark(label = "", &block)
  @bm ||= []
  @bm << Benchmark.measure("CV #{@current_cv} #{label}", &block)
end

#data_set ⇒ Object

This method is abstract.

Override this method to return an array.

A silly method meant to be overriden. should return an array, which will be then split up for cross-validating.



192
193
194
 
# File 'lib/experiment/base.rb', line 192

def data_set
  (1..cvs).to_a
end

#done? ⇒ Boolean

Is the experiment done.

Returns:

  • (Boolean) 


54
55
56
 
# File 'lib/experiment/base.rb', line 54

def done?
  @done
end

#measure(label = "", weight = nil, &block) ⇒ Object

Use this every time you want to do a measurement. It will be put on the record file and benchmarked automatically.

Parameters:

  • weight (Integer) (defaults to: nil) —

    Used for calculating Notify::step. It should be an integer denoting how many such measurements you wish to do.



127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/experiment/base.rb', line 127

def measure(label = "", weight = nil, &block)
  out = ""
  benchmark label do
    out = yield
  end
  if out.is_a? String
    @ouptut_file << out
  else
    YAML::dump(out, @ouptut_file)
  end
  Notify::step(@experiment, @current_cv, 1.0/weight) unless weight.nil?
end

#normal_run!(cv) ⇒ Object

runs the whole experiment, called by the framework



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/experiment/base.rb', line 82

def normal_run!(cv)
  @cvs = cv || 1
   @results = {}
  Notify.started @experiment
   split_up_data
  write_dir!
  @cvs.times do |cv_num|
    @bm = []
    @current_cv = cv_num
    File.open(@dir + "/raw-#{cv_num}.txt", "w") do |output|
      @ouptut_file = output
        run_the_experiment
    end
    array_merge @results, analyze_result!(@dir + "/raw-#{cv_num}.txt", @dir + "/analyzed-#{cv_num}.txt")
    write_performance!
    Notify.cv_done @experiment, cv_num
  end
  summarize_performance!
  summarize_results! @results
  specification!
  cleanup!
  Notify.completed @experiment
  puts File.read(@dir + "/summary.mmd") if @options.summary
end

#summarize_results!(results) ⇒ Object

creates a summary of the results and writes to ‘summary.mmd’



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/experiment/base.rb', line 154

def summarize_results!(results)
  File.open(@dir + '/results.yaml', 'w' ) do |out|
    YAML.dump(results, out)
  end
  # create an array of arrays
  res = results.keys.map do |key| 
    # calculate stats
    a = results[key]
    if a.all? {|el| el.is_a? Numeric }
      [key] + a + [Stats::mean(a), Stats::standard_deviation(a)]
   else
     [key] + a + ["--", "--"]
    end
 end
 
 ls = results.keys.map{|v| [7, v.to_s.length].max }
  
  ls = ["Std Deviation".length] + ls
  res = header_column + res
  res = res.transpose
  out = build_table res, ls
  File.open(@dir + "/summary.mmd", 'w') do |f|
    f << "## Results for #{@experiment} ##\n\n"
    f << out
 end
  #results = results.reduce({}) do |tot, res|
  # cv = res.delete :cv
  # tot.merge Hash[res.to_a.map {|a| ["cv_#{cv}_#{a.first}".to_sym, a.last]}]
  #end
  #FasterCSV.open("./results/all.csv", "a") do |csv|
  #  csv << results.to_a.sort_by{|a| a.first.to_s}.map(&:last)
  #end
end

#test_data ⇒ Array

Returns the portion of the #data_set that corresponds to the current cross validation number.

Returns:

  • (Array) 


110
111
112
 
# File 'lib/experiment/base.rb', line 110

def test_data
  @data[@current_cv]
end

#training_data ⇒ Array

Returns the #data_set that without the #test_data.

Returns:

  • (Array) 


116
117
118
 
# File 'lib/experiment/base.rb', line 116

def training_data
  (@data - test_data).flatten
end