Class: DerailedBenchmarks::StatsFromDir

Inherits:
Object
  • Object
show all
Defined in:
lib/derailed_benchmarks/stats_from_dir.rb

Overview

A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant

Example:

branch_info = {}
branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)

stats.newest.average  # => 10.5
stats.oldest.average  # => 11.0
stats.significant?    # => true
stats.x_faster        # => "1.0476"

Constant Summary collapse

FORMAT = 
"%0.4f"

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(input) ⇒ StatsFromDir

Returns a new instance of StatsFromDir.



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 37

def initialize(input)
  @files = []

  if input.is_a?(Hash)
    hash = input
    hash.each do |branch, info_hash|
      file = info_hash.fetch(:file)
      desc = info_hash.fetch(:desc)
      time = info_hash.fetch(:time)
      short_sha = info_hash[:short_sha]
      @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
    end
  else
    input.each do |commit|
      @files << StatsForFile.new(
        file: commit.file,
        desc: commit.desc,
        time: commit.time,
        name: commit.ref,
        short_sha: commit.short_sha
      )
    end
  end
  @files.sort_by! { |f| f.time }
  @oldest = @files.first
  @newest = @files.last
end

Instance Attribute Details

#newest ⇒ Object (readonly)

Returns the value of attribute newest.



35
36
37
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 35

def newest
  @newest
end

#oldest ⇒ Object (readonly)

Returns the value of attribute oldest.



35
36
37
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 35

def oldest
  @oldest
end

#stats ⇒ Object (readonly)

Returns the value of attribute stats.



35
36
37
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 35

def stats
  @stats
end

Instance Method Details

#align ⇒ Object 



123
124
125
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 123

def align
  " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
 


147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 147

def banner(io = $stdout)
  return if @files.detect(&:empty?)

  io.puts
  if significant?
    io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
  else
    io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
  end
  io.puts
  io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
  io.puts "  #{change_direction} by:"
  io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
  io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
  io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
  io.puts
  io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
  io.puts "Samples: #{newest.values.length}"
  io.puts
  io.puts "Test type: Kolmogorov Smirnov"
  io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
  io.puts "Is significant? (max > critical): #{significant?}"
  io.puts "D critical: #{d_critical}"
  io.puts "D max: #{d_max}"

  histogram(io)

  io.puts
end

#call ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 65

def call
  @files.each(&:call)

  return self if @files.detect(&:empty?)

  stats_95 = statistical_test(confidence: 95)

  # If default check is good, see if we also pass a more rigorous test
  # if so, then use the more rigourous test
  if stats_95[:alternative]
    stats_99 = statistical_test(confidence: 99)
    @stats = stats_99 if stats_99[:alternative]
  end
  @stats ||= stats_95

  self
end

#change_direction ⇒ Object 



115
116
117
118
119
120
121
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 115

def change_direction
  if faster?
    "FASTER 🚀🚀🚀"
  else
    "SLOWER 🐢🐢🐢"
  end
end

#d_critical ⇒ Object 



99
100
101
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 99

def d_critical
  @stats[:d_critical].to_f
end

#d_max ⇒ Object 



95
96
97
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 95

def d_max
  @stats[:d_max].to_f
end

#faster? ⇒ Boolean

Returns:

  • (Boolean) 


107
108
109
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 107

def faster?
  newest.median < oldest.median
end

#histogram(io = $stdout) ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 127

def histogram(io = $stdout)
  dual_histogram = MiniHistogram.dual_plot do |a, b|
    a.values = newest.values
    a.options = {
      title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
      xlabel: "# of runs in range"
    }
    b.values = oldest.values
    b.options = {
      title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
      xlabel: "# of runs in range"
    }
  end

  io.puts
  io.puts "Histograms (time ranges are in seconds):"
  io.puts(dual_histogram)
  io.puts
end

#percent_faster ⇒ Object 



111
112
113
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 111

def percent_faster
  (((oldest.median - newest.median) / oldest.median).to_f  * 100)
end

#significant? ⇒ Boolean

Returns:

  • (Boolean) 


91
92
93
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 91

def significant?
  @stats[:alternative]
end

#statistical_test(series_1 = oldest.values, series_2 = newest.values, confidence: 95) ⇒ Object 



83
84
85
86
87
88
89
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 83

def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
  StatisticalTest::KSTest.two_samples(
    group_one: series_1,
    group_two: series_2,
    alpha: (100 - confidence) / 100.0
  )
end

#x_faster ⇒ Object 



103
104
105
 
# File 'lib/derailed_benchmarks/stats_from_dir.rb', line 103

def x_faster
  (oldest.median/newest.median).to_f
end