Class: DerailedBenchmarks::StatsFromDir

Inherits:
Object
  • Object
show all
Defined in:
lib/derailed_benchmarks/stats_from_dir.rb

Overview

A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant

Example:

branch_info = {}
branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)

stats.newest.average  # => 10.5
stats.oldest.average  # => 11.0
stats.significant?    # => true
stats.x_faster        # => "1.0476"

Constant Summary collapse

FORMAT = 
"%0.4f"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ StatsFromDir

Returns a new instance of StatsFromDir.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 32

def initialize(input)
  @files = []

  if input.is_a?(Hash)
    hash = input
    hash.each do |branch, info_hash|
      file = info_hash.fetch(:file)
      desc = info_hash.fetch(:desc)
      time = info_hash.fetch(:time)
      short_sha = info_hash[:short_sha]
      @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
    end
  else
    input.each do |commit|
      @files << StatsForFile.new(
        file: commit.file,
        desc: commit.desc,
        time: commit.time,
        name: commit.ref,
        short_sha: commit.short_sha
      )
    end
  end
  @files.sort_by! { |f| f.time }
  @oldest = @files.first
  @newest = @files.last
end

Instance Attribute Details

#newest ⇒ Object (readonly)

Returns the value of attribute newest.



30
31
32
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 30

def newest
  @newest
end

#oldest ⇒ Object (readonly)

Returns the value of attribute oldest.



30
31
32
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 30

def oldest
  @oldest
end

#stats ⇒ Object (readonly)

Returns the value of attribute stats.



30
31
32
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 30

def stats
  @stats
end

Instance Method Details

#align ⇒ Object 



118
119
120
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 118

def align
  " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
 


142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 142

def banner(io = $stdout)
  return if @files.detect(&:empty?)

  io.puts
  if significant?
    io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
  else
    io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
  end
  io.puts
  io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
  io.puts "  #{change_direction} by:"
  io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
  io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
  io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
  io.puts
  io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
  io.puts "Samples: #{newest.values.length}"
  io.puts
  io.puts "Test type: Kolmogorov Smirnov"
  io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
  io.puts "Is significant? (max > critical): #{significant?}"
  io.puts "D critical: #{d_critical}"
  io.puts "D max: #{d_max}"

  histogram(io)

  io.puts
end

#call ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 60

def call
  @files.each(&:call)

  return self if @files.detect(&:empty?)

  stats_95 = statistical_test(confidence: 95)

  # If default check is good, see if we also pass a more rigorous test
  # if so, then use the more rigourous test
  if stats_95[:alternative]
    stats_99 = statistical_test(confidence: 99)
    @stats = stats_99 if stats_99[:alternative]
  end
  @stats ||= stats_95

  self
end

#change_direction ⇒ Object 



110
111
112
113
114
115
116
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 110

def change_direction
  if faster?
    "FASTER 🚀🚀🚀"
  else
    "SLOWER 🐢🐢🐢"
  end
end

#d_critical ⇒ Object 



94
95
96
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 94

def d_critical
  @stats[:d_critical].to_f
end

#d_max ⇒ Object 



90
91
92
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 90

def d_max
  @stats[:d_max].to_f
end

#faster? ⇒ Boolean

Returns:

  • (Boolean) 


102
103
104
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 102

def faster?
  newest.median < oldest.median
end

#histogram(io = $stdout) ⇒ Object 



122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 122

def histogram(io = $stdout)
  dual_histogram = MiniHistogram.dual_plot do |a, b|
    a.values = newest.values
    a.options = {
      title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
      xlabel: "# of runs in range"
    }
    b.values = oldest.values
    b.options = {
      title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
      xlabel: "# of runs in range"
    }
  end

  io.puts
  io.puts "Histograms (time ranges are in seconds):"
  io.puts(dual_histogram)
  io.puts
end

#percent_faster ⇒ Object 



106
107
108
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 106

def percent_faster
  (((oldest.median - newest.median) / oldest.median).to_f  * 100)
end

#significant? ⇒ Boolean

Returns:

  • (Boolean) 


86
87
88
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 86

def significant?
  @stats[:alternative]
end

#statistical_test(series_1 = oldest.values, series_2 = newest.values, confidence: 95) ⇒ Object 



78
79
80
81
82
83
84
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 78

def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
  StatisticalTest::KSTest.two_samples(
    group_one: series_1,
    group_two: series_2,
    alpha: (100 - confidence) / 100.0
  )
end

#x_faster ⇒ Object 



98
99
100
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 98

def x_faster
  (oldest.median/newest.median).to_f
end