Class: DerailedBenchmarks::StatsFromDir

Inherits:
Object
  • Object
show all
Defined in:
lib/derailed_benchmarks/stats_from_dir.rb

Overview

A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant

Example:

branch_info = {}
branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)

stats.newest.average  # => 10.5
stats.oldest.average  # => 11.0
stats.significant?    # => true
stats.x_faster        # => "1.0476"

Constant Summary collapse

FORMAT = 
"%0.4f"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(hash) ⇒ StatsFromDir

Returns a new instance of StatsFromDir.



29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 29

def initialize(hash)
  @files = []

  hash.each do |branch, info_hash|
    file = info_hash.fetch(:file)
    desc = info_hash.fetch(:desc)
    time = info_hash.fetch(:time)
    @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
  end
  @files.sort_by! { |f| f.time }
  @oldest = @files.first
  @newest = @files.last
end

Instance Attribute Details

#newest ⇒ Object (readonly)

Returns the value of attribute newest.



27
28
29
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 27

def newest
  @newest
end

#oldest ⇒ Object (readonly)

Returns the value of attribute oldest.



27
28
29
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 27

def oldest
  @oldest
end

#stats ⇒ Object (readonly)

Returns the value of attribute stats.



27
28
29
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 27

def stats
  @stats
end

Instance Method Details

#align ⇒ Object 



99
100
101
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 99

def align
  " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
 


103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 103

def banner(io = Kernel)
  io.puts
  if significant?
    io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
  else
    io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
  end
  io.puts
  io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
  io.puts "  #{change_direction} by:"
  io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
  io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
  io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
  io.puts
  io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
  io.puts "Samples: #{newest.values.length}"
  io.puts
  io.puts "Test type: Kolmogorov Smirnov"
  io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
  io.puts "Is significant? (max > critical): #{significant?}"
  io.puts "D critical: #{d_critical}"
  io.puts "D max: #{d_max}"
  io.puts
end

#call ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 43

def call
  @files.each(&:call)

  stats_95 = statistical_test(confidence: 95)

  # If default check is good, see if we also pass a more rigorous test
  # if so, then use the more rigourous test
  if stats_95[:alternative]
    stats_99 = statistical_test(confidence: 99)
    @stats = stats_99 if stats_99[:alternative]
  end
  @stats ||= stats_95

  self
end

#change_direction ⇒ Object 



91
92
93
94
95
96
97
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 91

def change_direction
  if faster?
    "FASTER 🚀🚀🚀"
  else
    "SLOWER 🐢🐢🐢"
  end
end

#d_critical ⇒ Object 



75
76
77
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 75

def d_critical
  @stats[:d_critical].to_f
end

#d_max ⇒ Object 



71
72
73
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 71

def d_max
  @stats[:d_max].to_f
end

#faster? ⇒ Boolean

Returns:

  • (Boolean) 


83
84
85
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 83

def faster?
  newest.median < oldest.median
end

#percent_faster ⇒ Object 



87
88
89
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 87

def percent_faster
  (((oldest.median - newest.median) / oldest.median).to_f  * 100)
end

#significant? ⇒ Boolean

Returns:

  • (Boolean) 


67
68
69
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 67

def significant?
  @stats[:alternative]
end

#statistical_test(series_1 = oldest.values, series_2 = newest.values, confidence: 95) ⇒ Object 



59
60
61
62
63
64
65
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 59

def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
  StatisticalTest::KSTest.two_samples(
    group_one: series_1,
    group_two: series_2,
    alpha: (100 - confidence) / 100.0
  )
end

#x_faster ⇒ Object 



79
80
81
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 79

def x_faster
  (oldest.median/newest.median).to_f
end