Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/noop_report.rb,
lib/benchmark/ips/job/stdout_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, NoopReport, StdoutReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05
POW_2_30 = 
1 << 30

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/benchmark/ips/job.rb', line 63

def initialize opts={}
  @list = []
  @run_single = false
  @json_path = false
  @compare = false
  @held_path = nil
  @held_results = nil

  @timing = Hash.new 1 # default to 1 in case warmup isn't run
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95

  self.quiet = false
end

Instance Attribute Details

#compare ⇒ Boolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean) —

    true if needs to run compare.



20
21
22
 
# File 'lib/benchmark/ips/job.rb', line 20

def compare
  @compare
end

#confidence ⇒ Integer

Confidence.

Returns:

  • (Integer) 


52
53
54
 
# File 'lib/benchmark/ips/job.rb', line 52

def confidence
  @confidence
end

#full_report ⇒ Report (readonly)

Report object containing information about the run.

Returns:

  • (Report) —

    the report object.



28
29
30
 
# File 'lib/benchmark/ips/job.rb', line 28

def full_report
  @full_report
end

#hold ⇒ Boolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean) 


24
25
26
 
# File 'lib/benchmark/ips/job.rb', line 24

def hold
  @hold
end

#iterations ⇒ Integer

Warmup and calculation iterations.

Returns:

  • (Integer) 


44
45
46
 
# File 'lib/benchmark/ips/job.rb', line 44

def iterations
  @iterations
end

#list ⇒ Array<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>) —

    list of entries



16
17
18
 
# File 'lib/benchmark/ips/job.rb', line 16

def list
  @list
end

#quiet ⇒ Boolean

Silence output

Returns:

  • (Boolean) 


56
57
58
 
# File 'lib/benchmark/ips/job.rb', line 56

def quiet
  @quiet
end

#stats ⇒ Object

Statistics model.

Returns:

  • (Object) 


48
49
50
 
# File 'lib/benchmark/ips/job.rb', line 48

def stats
  @stats
end

#suite ⇒ Benchmark::IPS::NoopSuite

Suite

Returns:



60
61
62
 
# File 'lib/benchmark/ips/job.rb', line 60

def suite
  @suite
end

#time ⇒ Integer

Calculation time setter and getter (in seconds).

Returns:

  • (Integer) 


40
41
42
 
# File 'lib/benchmark/ips/job.rb', line 40

def time
  @time
end

#timing ⇒ Hash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash) 


32
33
34
 
# File 'lib/benchmark/ips/job.rb', line 32

def timing
  @timing
end

#warmup ⇒ Integer

Warmup time setter and getter (in seconds).

Returns:

  • (Integer) 


36
37
38
 
# File 'lib/benchmark/ips/job.rb', line 36

def warmup
  @warmup
end

Instance Method Details

#all_results_have_been_run? ⇒ Boolean

Returns:

  • (Boolean) 


236
237
238
 
# File 'lib/benchmark/ips/job.rb', line 236

def all_results_have_been_run?
  @full_report.entries.size == @list.size
end

#clear_held_results ⇒ Object 



240
241
242
 
# File 'lib/benchmark/ips/job.rb', line 240

def clear_held_results
  File.delete @held_path if File.exist?(@held_path)
end

#compare! ⇒ Object

Run comparison utility.



120
121
122
 
# File 'lib/benchmark/ips/job.rb', line 120

def compare!
  @compare = true
end

#compare? ⇒ Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean) —

    Need to compare?



115
116
117
 
# File 'lib/benchmark/ips/job.rb', line 115

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash) —

    a customizable set of options

  • iterations (Hash) —

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer) —

    Warmup time.

  • :time (Integer) —

    Calculation time.



90
91
92
93
94
95
96
97
98
99
 
# File 'lib/benchmark/ips/job.rb', line 90

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @suite = opts[:suite] if opts[:suite]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
  self.quiet = opts[:quiet] if opts.key?(:quiet)
  self.suite = opts[:suite]
end

#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String) —

    Report item label.

  • measured_us (Integer) —

    Measured time in microsecond.

  • iter (Integer) —

    Iterations.

  • samples (Array<Float>) —

    Sampled iterations per second.

  • cycles (Integer) —

    Number of Cycles.

Returns:



391
392
393
 
# File 'lib/benchmark/ips/job.rb', line 391

def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end

#create_stats(samples) ⇒ Object 



363
364
365
366
367
368
369
370
371
372
 
# File 'lib/benchmark/ips/job.rb', line 363

def create_stats(samples)
  case @stats
    when :sd
      Stats::SD.new(samples)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float) —

    Each iteration’s time in ms.

  • iters (Integer) —

    Iterations.

Returns:

  • (Integer) —

    Cycles per 100ms.



188
189
190
191
 
# File 'lib/benchmark/ips/job.rb', line 188

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end

#generate_json ⇒ Object

Generate json from @full_report.



380
381
382
 
# File 'lib/benchmark/ips/job.rb', line 380

def generate_json
  @full_report.generate_json @json_path if json?
end

#hold!(held_path) ⇒ Object

Hold after each iteration.

Parameters:

  • held_path (String) —

    File name to store hold file.



132
133
134
135
 
# File 'lib/benchmark/ips/job.rb', line 132

def hold!(held_path)
  @held_path = held_path
  @run_single = true
end

#hold? ⇒ Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean) —

    Need to hold results between multiple Ruby invocations?



126
127
128
 
# File 'lib/benchmark/ips/job.rb', line 126

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "") —

    Label of benchmarked code.

  • str (String) (defaults to: nil) —

    Code to be benchmarked.

  • blk (Proc) —

    Code to be benchmarked.

Raises:

  • (ArgumentError) —

    Raises if str and blk are both present.

  • (ArgumentError) —

    Raises if str and blk are both absent.



170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/benchmark/ips/job.rb', line 170

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer) —

    Cycles.

  • time_us (Integer) —

    Time in microsecond.

Returns:

  • (Float) —

    Iteration per second.



206
207
208
 
# File 'lib/benchmark/ips/job.rb', line 206

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Generate json to given path, defaults to “data.json”.



160
161
162
 
# File 'lib/benchmark/ips/job.rb', line 160

def json!(path="data.json")
  @json_path = path
end

#json? ⇒ Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean) —

    Need to generate json?



155
156
157
 
# File 'lib/benchmark/ips/job.rb', line 155

def json?
  !!@json_path
end

#load_held_results ⇒ Object 



210
211
212
213
214
215
216
217
218
219
 
# File 'lib/benchmark/ips/job.rb', line 210

def load_held_results
  return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
  require "json"
  @held_results = {}
  JSON.load(IO.read(@held_path)).each do |result|
    @held_results[result['item']] = result
    create_report(result['item'], result['measured_us'], result['iter'],
                  create_stats(result['samples']), result['cycles'])
  end
end

#reporter(quiet:) ⇒ Object 



109
110
111
 
# File 'lib/benchmark/ips/job.rb', line 109

def reporter(quiet:)
  quiet ? NoopReport.new : StdoutReport.new
end

#run ⇒ Object 



244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
 
# File 'lib/benchmark/ips/job.rb', line 244

def run
  if @warmup && @warmup != 0 then
    @stdout.start_warming
    @iterations.times do
      run_warmup
    end
  end

  @stdout.start_running

  @iterations.times do |n|
    run_benchmark
  end

  @stdout.footer
end

#run_benchmark ⇒ Object

Run calculation.



309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
 
# File 'lib/benchmark/ips/job.rb', line 309

def run_benchmark
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.running item.label, @time
    @stdout.running item.label, @time

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time

    begin
      before = Timing.now
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end while Timing.now < target

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @stdout.add_report rep, caller(1).first
    @suite.add_report rep, caller(1).first

    break if run_single?
  end
end

#run_comparison ⇒ Object

Run comparison of entries in @full_report.



375
376
377
 
# File 'lib/benchmark/ips/job.rb', line 375

def run_comparison
  @full_report.run_comparison if compare?
end

#run_single? ⇒ Boolean

Return true if items are to be run one at a time. For the traditional hold, this is true

Returns:

  • (Boolean) —

    Run just a single item?



149
150
151
 
# File 'lib/benchmark/ips/job.rb', line 149

def run_single?
  @run_single
end

#run_warmup ⇒ Object

Run warmup.



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
 
# File 'lib/benchmark/ips/job.rb', line 262

def run_warmup
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.warming item.label, @warmup
    @stdout.warming item.label, @warmup

    Timing.clean_env

    # Run for up to half of the configured warmup time with an increasing
    # number of cycles to reduce overhead and improve accuracy.
    # This also avoids running with a constant number of cycles, which a
    # JIT might speculate on and then have to recompile in #run_benchmark.
    before = Timing.now
    target = Timing.add_second before, @warmup / 2.0

    cycles = 1
    begin
      t0 = Timing.now
      item.call_times cycles
      t1 = Timing.now
      warmup_iter = cycles
      warmup_time_us = Timing.time_us(t0, t1)

      # If the number of cycles would go outside the 32-bit signed integers range
      # then exit the loop to avoid overflows and start the 100ms warmup runs
      break if cycles >= POW_2_30
      cycles *= 2
    end while Timing.now + warmup_time_us * 2 < target

    cycles = cycles_per_100ms warmup_time_us, warmup_iter
    @timing[item] = cycles

    # Run for the remaining of warmup in a similar way as #run_benchmark.
    target = Timing.add_second before, @warmup
    while Timing.now + MICROSECONDS_PER_100MS < target
      item.call_times cycles
    end

    @stdout.warmup_stats warmup_time_us, @timing[item]
    @suite.warmup_stats warmup_time_us, @timing[item]

    break if run_single?
  end
end

#save!(held_path) ⇒ Object

Save interim results. Similar to hold, but all reports are run The report label must change for each invocation. One way to achieve this is to include the version in the label.

Parameters:

  • held_path (String) —

    File name to store hold file.



141
142
143
144
 
# File 'lib/benchmark/ips/job.rb', line 141

def save!(held_path)
  @held_path = held_path
  @run_single = false
end

#save_held_results ⇒ Object 



221
222
223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/benchmark/ips/job.rb', line 221

def save_held_results
  return unless @held_path
  require "json"
  data = full_report.entries.map { |e|
    {
      'item' => e.label,
      'measured_us' => e.microseconds,
      'iter' => e.iterations,
      'samples' => e.samples,
      'cycles' => e.measurement_cycle
    }
  }
  IO.write(@held_path, JSON.generate(data) << "\n")
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time) —

    time.

  • after (Time) —

    time.

Returns:

  • (Float) —

    Time difference of before and after.



197
198
199
 
# File 'lib/benchmark/ips/job.rb', line 197

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end