Class: Benchmark::IPS::Job
- Inherits:
-
Object
- Object
- Benchmark::IPS::Job
- Defined in:
- lib/project/ips/job.rb,
lib/project/ips/job/entry.rb,
lib/project/ips/job/stdout_report.rb
Overview
Benchmark jobs.
Defined Under Namespace
Classes: Entry, StdoutReport
Constant Summary collapse
- MICROSECONDS_PER_100MS =
Microseconds per 100 millisecond.
100_000
- MICROSECONDS_PER_SECOND =
Microseconds per second.
Timing::MICROSECONDS_PER_SECOND
- MAX_TIME_SKEW =
The percentage of the expected runtime to allow before reporting a weird runtime
0.05
Instance Attribute Summary collapse
-
#compare ⇒ Boolean
readonly
Determining whether to run comparison utility.
-
#confidence ⇒ Integer
Confidence.
-
#full_report ⇒ Report
readonly
Report object containing information about the run.
-
#hold ⇒ Boolean
Determining whether to hold results between Ruby invocations.
-
#iterations ⇒ Integer
Warmup and calculation iterations.
-
#list ⇒ Array<Entry>
readonly
Two-element arrays, consisting of label and block pairs.
-
#stats ⇒ Object
Statistics model.
-
#time ⇒ Integer
Calculation time setter and getter (in seconds).
-
#timing ⇒ Hash
readonly
Storing Iterations in time period.
-
#warmup ⇒ Integer
Warmup time setter and getter (in seconds).
Instance Method Summary collapse
-
#compare! ⇒ Object
Set @compare to true.
-
#compare? ⇒ Boolean
Return true if job needs to be compared.
-
#config(opts) ⇒ Object
Job configuration options, set @warmup and @time.
-
#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry
Create report by add entry to @full_report.
- #create_stats(samples) ⇒ Object
-
#cycles_per_100ms(time_msec, iters) ⇒ Integer
Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.
-
#generate_json ⇒ Object
Generate json from @full_report.
- #held_results? ⇒ Boolean
-
#hold!(held_path) ⇒ Object
Set @hold to true.
-
#hold? ⇒ Boolean
Return true if results are held while multiple Ruby invocations.
-
#initialize(opts = {}) ⇒ Job
constructor
Instantiate the Benchmark::IPS::Job.
-
#item(label = "", str = nil, &blk) ⇒ Object
(also: #report)
Registers the given label and block pair in the job list.
-
#iterations_per_sec(cycles, time_us) ⇒ Float
Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.
-
#json!(path = "data.json") ⇒ Object
Set @json_path to given path, defaults to “data.json”.
-
#json? ⇒ Boolean
Return true if job needs to generate json.
- #load_held_results ⇒ Object
- #run ⇒ Object
-
#run_benchmark ⇒ Object
Run calculation.
-
#run_comparison ⇒ Object
Run comparison of entries in @full_report.
-
#run_warmup ⇒ Object
Run warmup.
-
#time_us(before, after) ⇒ Float
Calculate the time difference of before and after in microseconds.
Constructor Details
#initialize(opts = {}) ⇒ Job
Instantiate the Benchmark::IPS::Job.
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
# File 'lib/project/ips/job.rb', line 56 def initialize opts={} @suite = opts[:suite] || nil @stdout = opts[:quiet] ? nil : StdoutReport.new @list = [] @compare = false @json_path = false @held_path = nil @held_results = nil @timing = Hash.new 1 # default to 1 in case warmup isn't run @full_report = Report.new # Default warmup and calculation time in seconds. @warmup = 2 @time = 5 @iterations = 1 # Default statistical model @stats = :sd @confidence = 95 end |
Instance Attribute Details
#compare ⇒ Boolean (readonly)
Determining whether to run comparison utility.
19 20 21 |
# File 'lib/project/ips/job.rb', line 19 def compare @compare end |
#confidence ⇒ Integer
Confidence.
51 52 53 |
# File 'lib/project/ips/job.rb', line 51 def confidence @confidence end |
#full_report ⇒ Report (readonly)
Report object containing information about the run.
27 28 29 |
# File 'lib/project/ips/job.rb', line 27 def full_report @full_report end |
#hold ⇒ Boolean
Determining whether to hold results between Ruby invocations
23 24 25 |
# File 'lib/project/ips/job.rb', line 23 def hold @hold end |
#iterations ⇒ Integer
Warmup and calculation iterations.
43 44 45 |
# File 'lib/project/ips/job.rb', line 43 def iterations @iterations end |
#list ⇒ Array<Entry> (readonly)
Two-element arrays, consisting of label and block pairs.
15 16 17 |
# File 'lib/project/ips/job.rb', line 15 def list @list end |
#stats ⇒ Object
Statistics model.
47 48 49 |
# File 'lib/project/ips/job.rb', line 47 def stats @stats end |
#time ⇒ Integer
Calculation time setter and getter (in seconds).
39 40 41 |
# File 'lib/project/ips/job.rb', line 39 def time @time end |
#timing ⇒ Hash (readonly)
Storing Iterations in time period.
31 32 33 |
# File 'lib/project/ips/job.rb', line 31 def timing @timing end |
#warmup ⇒ Integer
Warmup time setter and getter (in seconds).
35 36 37 |
# File 'lib/project/ips/job.rb', line 35 def warmup @warmup end |
Instance Method Details
#compare! ⇒ Object
Set @compare to true.
98 99 100 |
# File 'lib/project/ips/job.rb', line 98 def compare! @compare = true end |
#compare? ⇒ Boolean
Return true if job needs to be compared.
93 94 95 |
# File 'lib/project/ips/job.rb', line 93 def compare? @compare end |
#config(opts) ⇒ Object
Job configuration options, set @warmup and @time.
82 83 84 85 86 87 88 89 |
# File 'lib/project/ips/job.rb', line 82 def config opts @warmup = opts[:warmup] if opts[:warmup] @time = opts[:time] if opts[:time] @suite = opts[:suite] if opts[:suite] @iterations = opts[:iterations] if opts[:iterations] @stats = opts[:stats] if opts[:stats] @confidence = opts[:confidence] if opts[:confidence] end |
#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry
Create report by add entry to @full_report.
346 347 348 |
# File 'lib/project/ips/job.rb', line 346 def create_report(label, measured_us, iter, samples, cycles) @full_report.add_entry label, measured_us, iter, samples, cycles end |
#create_stats(samples) ⇒ Object
318 319 320 321 322 323 324 325 326 327 |
# File 'lib/project/ips/job.rb', line 318 def create_stats(samples) case @stats when :sd Stats::SD.new(samples) when :bootstrap Stats::Bootstrap.new(samples, @confidence) else raise "unknown stats #{@stats}" end end |
#cycles_per_100ms(time_msec, iters) ⇒ Integer
Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.
148 149 150 151 |
# File 'lib/project/ips/job.rb', line 148 def cycles_per_100ms time_msec, iters cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i cycles <= 0 ? 1 : cycles end |
#generate_json ⇒ Object
Generate json from @full_report.
335 336 337 |
# File 'lib/project/ips/job.rb', line 335 def generate_json @full_report.generate_json @json_path if json? end |
#held_results? ⇒ Boolean
170 171 172 |
# File 'lib/project/ips/job.rb', line 170 def held_results? File.exist?(@held_path) end |
#hold!(held_path) ⇒ Object
Set @hold to true.
109 110 111 |
# File 'lib/project/ips/job.rb', line 109 def hold!(held_path) @held_path = held_path end |
#hold? ⇒ Boolean
Return true if results are held while multiple Ruby invocations
104 105 106 |
# File 'lib/project/ips/job.rb', line 104 def hold? !!@held_path end |
#item(label = "", str = nil, &blk) ⇒ Object Also known as: report
Registers the given label and block pair in the job list.
130 131 132 133 134 135 136 137 138 139 140 |
# File 'lib/project/ips/job.rb', line 130 def item(label="", str=nil, &blk) # :yield: if blk and str raise ArgumentError, "specify a block and a str, but not both" end action = str || blk raise ArgumentError, "no block or string" unless action @list.push Entry.new(label, action) self end |
#iterations_per_sec(cycles, time_us) ⇒ Float
Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.
166 167 168 |
# File 'lib/project/ips/job.rb', line 166 def iterations_per_sec cycles, time_us MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f) end |
#json!(path = "data.json") ⇒ Object
Set @json_path to given path, defaults to “data.json”.
120 121 122 |
# File 'lib/project/ips/job.rb', line 120 def json!(path="data.json") @json_path = path end |
#json? ⇒ Boolean
Return true if job needs to generate json.
115 116 117 |
# File 'lib/project/ips/job.rb', line 115 def json? !!@json_path end |
#load_held_results ⇒ Object
174 175 176 177 178 179 180 |
# File 'lib/project/ips/job.rb', line 174 def load_held_results require "json" @held_results = Hash[File.open(@held_path).map { |line| result = JSON.parse(line) [result['item'], result] }] end |
#run ⇒ Object
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 |
# File 'lib/project/ips/job.rb', line 182 def run if @warmup && @warmup != 0 then @stdout.start_warming if @stdout @iterations.times do run_warmup end end @stdout.start_running if @stdout held = nil @iterations.times do |n| held = run_benchmark end @stdout. if @stdout if held puts puts 'Pausing here -- run Ruby again to measure the next benchmark...' end end |
#run_benchmark ⇒ Object
Run calculation.
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 |
# File 'lib/project/ips/job.rb', line 240 def run_benchmark @list.each do |item| if hold? && @held_results && @held_results.key?(item.label) result = @held_results[item.label] create_report(item.label, result['measured_us'], result['iter'], create_stats(result['samples']), result['cycles']) next end @suite.running item.label, @time if @suite @stdout.running item.label, @time if @stdout Timing.clean_env iter = 0 measurements_us = [] # Running this number of cycles should take around 100ms. cycles = @timing[item] target = Timing.add_second Timing.now, @time while (before = Timing.now) < target item.call_times cycles after = Timing.now # If for some reason the timing said this took no time (O_o) # then ignore the iteration entirely and start another. iter_us = Timing.time_us before, after next if iter_us <= 0.0 iter += cycles measurements_us << iter_us end final_time = before measured_us = measurements_us.inject(:+) samples = measurements_us.map { |time_us| iterations_per_sec cycles, time_us } rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles) if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW) rep.show_total_time! end @stdout.add_report rep, caller(1).first if @stdout @suite.add_report rep, caller(1).first if @suite if hold? && item != @list.last File.open @held_path, "a" do |f| require "json" f.write JSON.generate({ :item => item.label, :measured_us => measured_us, :iter => iter, :samples => samples, :cycles => cycles }) f.write "\n" end return true end end if hold? && @full_report.entries.size == @list.size File.delete @held_path if File.exist?(@held_path) end false end |
#run_comparison ⇒ Object
Run comparison of entries in @full_report.
330 331 332 |
# File 'lib/project/ips/job.rb', line 330 def run_comparison @full_report.run_comparison if compare? end |
#run_warmup ⇒ Object
Run warmup.
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 |
# File 'lib/project/ips/job.rb', line 207 def run_warmup @list.each do |item| next if hold? && @held_results && @held_results.key?(item.label) @suite.warming item.label, @warmup if @suite @stdout.warming item.label, @warmup if @stdout Timing.clean_env before = Timing.now target = Timing.add_second before, @warmup warmup_iter = 0 while Timing.now < target item.call_times(1) warmup_iter += 1 end after = Timing.now warmup_time_us = Timing.time_us(before, after) @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout @suite.warmup_stats warmup_time_us, @timing[item] if @suite break if hold? end end |
#time_us(before, after) ⇒ Float
Calculate the time difference of before and after in microseconds.
157 158 159 |
# File 'lib/project/ips/job.rb', line 157 def time_us before, after (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND end |