Module: Benchmark::Experiment

Included in:
Benchmark
Defined in:
lib/benchmark/lab.rb,
lib/benchmark/lab/version.rb,
lib/benchmark/lab/mann_whitney_u_test.rb,
lib/benchmark/lab/descriptive_statistics.rb

Defined Under Namespace

Modules: MannWhitneyUTest Classes: DescriptiveStatistics

Constant Summary collapse

MEASURED_TIMES = 
{
  utime: 'user',
  stime: 'system',
  total: 'total',
  real: 'real'
}
VERSION = 
'0.0.1'

Instance Method Summary collapse

Instance Method Details

#aggregate_and_rank(jsons) ⇒ Object 



72
73
74
75
76
 
# File 'lib/benchmark/lab.rb', line 72

def aggregate_and_rank(jsons)
  return if jsons.empty?
  all_stats = jsons.inject({}) { |elem, hsh| hsh.merge(elem) }
  rank(all_stats)
end

#experiment(sample_size, &blk) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
 
# File 'lib/benchmark/lab.rb', line 78

def experiment(sample_size, &blk)
  all_stats = JSON.parse(observe_and_summarize(sample_size, &blk))
  print_stats(all_stats)

  best, is_the_best_significative = rank(all_stats)

  puts "The best \"#{best['label']}\" is #{is_the_best_significative ? '' : 'not '}significantly (95%) better (total time)."

  all_stats
end

#iterative_experiment ⇒ Object 



105
106
 
# File 'lib/benchmark/lab.rb', line 105

def iterative_experiment
end

#observe_and_summarize(sample_size) {|job| ... } ⇒ Object

Yields:

  • (job) 


64
65
66
67
68
69
70
 
# File 'lib/benchmark/lab.rb', line 64

def observe_and_summarize(sample_size, &blk)
  job = Job.new(0)
  yield(job)
  job.observe_and_summarize(sample_size)
  all_stats = job.list.map{ |label, _, _, stats| [label, stats] }.to_h
  all_stats.to_json
end

#rank(all_stats, alpha = 0.05) ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/benchmark/lab.rb', line 89

def rank(all_stats, alpha = 0.05)
  ranked = all_stats.map do |label, stats|
    total = stats.select{ |stat| stat['name'] == 'total' }.first
    total['label'] = label
    total
  end.sort_by { |stat| stat['median'] }
  is_h0_rejected = true
  if all_stats.size > 1
    z = Benchmark::Experiment::MannWhitneyUTest::calculate_z(ranked.first['sample'], ranked[1]['sample'])
    p_value = Benchmark::Experiment::MannWhitneyUTest::calculate_probability_z(z)
    is_h0_rejected = Benchmark::Experiment::MannWhitneyUTest::is_null_hypothesis_rejected?(p_value, alpha)
  end

  return ranked.first, is_h0_rejected
end