Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/stdout_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, StdoutReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05
POW_2_30 =
1 << 30

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.

Parameters:

  • opts (Hash) (defaults to: {})

    a customizable set of options

Options Hash (opts):

  • (nil) (Benchmark::Suite)

    :suite Specify Benchmark::Suite.

  • (false) (Boolean)

    :quiet Suppress the printing of information.


57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
# File 'lib/benchmark/ips/job.rb', line 57

def initialize opts={}
  @suite = opts[:suite] || nil
  @stdout = opts[:quiet] ? nil : StdoutReport.new
  @list = []
  @compare = false
  @run_single = false
  @json_path = false
  @held_path = nil
  @held_results = nil

  @timing = Hash.new 1 # default to 1 in case warmup isn't run
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95
end

Instance Attribute Details

#compareBoolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean)

    true if needs to run compare.


20
21
22
# File 'lib/benchmark/ips/job.rb', line 20

def compare
  @compare
end

#confidenceInteger

Confidence.

Returns:

  • (Integer)

52
53
54
# File 'lib/benchmark/ips/job.rb', line 52

def confidence
  @confidence
end

#full_reportReport (readonly)

Report object containing information about the run.

Returns:

  • (Report)

    the report object.


28
29
30
# File 'lib/benchmark/ips/job.rb', line 28

def full_report
  @full_report
end

#holdBoolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean)

24
25
26
# File 'lib/benchmark/ips/job.rb', line 24

def hold
  @hold
end

#iterationsInteger

Warmup and calculation iterations.

Returns:

  • (Integer)

44
45
46
# File 'lib/benchmark/ips/job.rb', line 44

def iterations
  @iterations
end

#listArray<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>)

    list of entries


16
17
18
# File 'lib/benchmark/ips/job.rb', line 16

def list
  @list
end

#statsObject

Statistics model.

Returns:

  • (Object)

48
49
50
# File 'lib/benchmark/ips/job.rb', line 48

def stats
  @stats
end

#timeInteger

Calculation time setter and getter (in seconds).

Returns:

  • (Integer)

40
41
42
# File 'lib/benchmark/ips/job.rb', line 40

def time
  @time
end

#timingHash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash)

32
33
34
# File 'lib/benchmark/ips/job.rb', line 32

def timing
  @timing
end

#warmupInteger

Warmup time setter and getter (in seconds).

Returns:

  • (Integer)

36
37
38
# File 'lib/benchmark/ips/job.rb', line 36

def warmup
  @warmup
end

Instance Method Details

#all_results_have_been_run?Boolean

Returns:

  • (Boolean)

216
217
218
# File 'lib/benchmark/ips/job.rb', line 216

def all_results_have_been_run?
  @full_report.entries.size == @list.size
end

#clear_held_resultsObject


220
221
222
# File 'lib/benchmark/ips/job.rb', line 220

def clear_held_results
  File.delete @held_path if File.exist?(@held_path)
end

#compare!Object

Run comparison utility.


100
101
102
# File 'lib/benchmark/ips/job.rb', line 100

def compare!
  @compare = true
end

#compare?Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean)

    Need to compare?


95
96
97
# File 'lib/benchmark/ips/job.rb', line 95

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash)

    a customizable set of options

  • iterations (Hash)

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer)

    Warmup time.

  • :time (Integer)

    Calculation time.


84
85
86
87
88
89
90
91
# File 'lib/benchmark/ips/job.rb', line 84

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @suite = opts[:suite] if opts[:suite]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
end

#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String)

    Report item label.

  • measured_us (Integer)

    Measured time in microsecond.

  • iter (Integer)

    Iterations.

  • samples (Array<Float>)

    Sampled iterations per second.

  • cycles (Integer)

    Number of Cycles.

Returns:


372
373
374
# File 'lib/benchmark/ips/job.rb', line 372

def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end

#create_stats(samples) ⇒ Object


344
345
346
347
348
349
350
351
352
353
# File 'lib/benchmark/ips/job.rb', line 344

def create_stats(samples)
  case @stats
    when :sd
      Stats::SD.new(samples)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float)

    Each iteration's time in ms.

  • iters (Integer)

    Iterations.

Returns:

  • (Integer)

    Cycles per 100ms.


168
169
170
171
# File 'lib/benchmark/ips/job.rb', line 168

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end

#generate_jsonObject

Generate json from @full_report.


361
362
363
# File 'lib/benchmark/ips/job.rb', line 361

def generate_json
  @full_report.generate_json @json_path if json?
end

#hold!(held_path) ⇒ Object

Hold after each iteration.

Parameters:

  • held_path (String)

    File name to store hold file.


112
113
114
115
# File 'lib/benchmark/ips/job.rb', line 112

def hold!(held_path)
  @held_path = held_path
  @run_single = true
end

#hold?Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean)

    Need to hold results between multiple Ruby invocations?


106
107
108
# File 'lib/benchmark/ips/job.rb', line 106

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "")

    Label of benchmarked code.

  • str (String) (defaults to: nil)

    Code to be benchmarked.

  • blk (Proc)

    Code to be benchmarked.

Raises:

  • (ArgumentError)

    Raises if str and blk are both present.

  • (ArgumentError)

    Raises if str and blk are both absent.


150
151
152
153
154
155
156
157
158
159
160
# File 'lib/benchmark/ips/job.rb', line 150

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer)

    Cycles.

  • time_us (Integer)

    Time in microsecond.

Returns:

  • (Float)

    Iteration per second.


186
187
188
# File 'lib/benchmark/ips/job.rb', line 186

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Generate json to given path, defaults to “data.json”.


140
141
142
# File 'lib/benchmark/ips/job.rb', line 140

def json!(path="data.json")
  @json_path = path
end

#json?Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean)

    Need to generate json?


135
136
137
# File 'lib/benchmark/ips/job.rb', line 135

def json?
  !!@json_path
end

#load_held_resultsObject


190
191
192
193
194
195
196
197
198
199
# File 'lib/benchmark/ips/job.rb', line 190

def load_held_results
  return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
  require "json"
  @held_results = {}
  JSON.load(IO.read(@held_path)).each do |result|
    @held_results[result['item']] = result
    create_report(result['item'], result['measured_us'], result['iter'],
                  create_stats(result['samples']), result['cycles'])
  end
end

#runObject


224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
# File 'lib/benchmark/ips/job.rb', line 224

def run
  if @warmup && @warmup != 0 then
    @stdout.start_warming if @stdout
    @iterations.times do
      run_warmup
    end
  end

  @stdout.start_running if @stdout

  @iterations.times do |n|
    run_benchmark
  end

  @stdout.footer if @stdout
end

#run_benchmarkObject

Run calculation.


291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
# File 'lib/benchmark/ips/job.rb', line 291

def run_benchmark
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.running item.label, @time if @suite
    @stdout.running item.label, @time if @stdout

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time

    while (before = Timing.now) < target
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @stdout.add_report rep, caller(1).first if @stdout
    @suite.add_report rep, caller(1).first if @suite

    break if run_single?
  end
end

#run_comparisonObject

Run comparison of entries in @full_report.


356
357
358
# File 'lib/benchmark/ips/job.rb', line 356

def run_comparison
  @full_report.run_comparison if compare?
end

#run_single?Boolean

Return true if items are to be run one at a time. For the traditional hold, this is true

Returns:

  • (Boolean)

    Run just a single item?


129
130
131
# File 'lib/benchmark/ips/job.rb', line 129

def run_single?
  @run_single
end

#run_warmupObject

Run warmup.


242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
# File 'lib/benchmark/ips/job.rb', line 242

def run_warmup
  @list.each do |item|
    next if run_single? && @held_results && @held_results.key?(item.label)

    @suite.warming item.label, @warmup if @suite
    @stdout.warming item.label, @warmup if @stdout

    Timing.clean_env

    # Run for up to half of the configured warmup time with an increasing
    # number of cycles to reduce overhead and improve accuracy.
    # This also avoids running with a constant number of cycles, which a
    # JIT might speculate on and then have to recompile in #run_benchmark.
    before = Timing.now
    target = Timing.add_second before, @warmup / 2.0

    cycles = 1
    warmup_iter = 1
    warmup_time_us = 0.0
    while Timing.now + warmup_time_us * 2 < target
      t0 = Timing.now
      item.call_times cycles
      t1 = Timing.now
      warmup_iter = cycles
      warmup_time_us = Timing.time_us(t0, t1)

      # If the number of cycles would go outside the 32-bit signed integers range
      # then exit the loop to avoid overflows and start the 100ms warmup runs
      break if cycles >= POW_2_30
      cycles *= 2
    end

    cycles = cycles_per_100ms warmup_time_us, warmup_iter
    @timing[item] = cycles

    # Run for the remaining of warmup in a similar way as #run_benchmark.
    target = Timing.add_second before, @warmup
    while Timing.now + MICROSECONDS_PER_100MS < target
      item.call_times cycles
    end

    @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
    @suite.warmup_stats warmup_time_us, @timing[item] if @suite

    break if run_single?
  end
end

#save!(held_path) ⇒ Object

Save interim results. Similar to hold, but all reports are run The report label must change for each invocation. One way to achieve this is to include the version in the label.

Parameters:

  • held_path (String)

    File name to store hold file.


121
122
123
124
# File 'lib/benchmark/ips/job.rb', line 121

def save!(held_path)
  @held_path = held_path
  @run_single = false
end

#save_held_resultsObject


201
202
203
204
205
206
207
208
209
210
211
212
213
214
# File 'lib/benchmark/ips/job.rb', line 201

def save_held_results
  return unless @held_path
  require "json"
  data = full_report.entries.map { |e|
    {
      'item' => e.label,
      'measured_us' => e.microseconds,
      'iter' => e.iterations,
      'samples' => e.samples,
      'cycles' => e.measurement_cycle
    }
  }
  IO.write(@held_path, JSON.generate(data) << "\n")
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time)

    time.

  • after (Time)

    time.

Returns:

  • (Float)

    Time difference of before and after.


177
178
179
# File 'lib/benchmark/ips/job.rb', line 177

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end