Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/stdout_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, StdoutReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.

Parameters:

  • opts (Hash) (defaults to: {}) —

    a customizable set of options

Options Hash (opts):

  • (nil) (Benchmark::Suite) —

    :suite Specify Benchmark::Suite.

  • (false) (Boolean) —

    :quiet Suppress the printing of information.



56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/benchmark/ips/job.rb', line 56

def initialize opts={}
  @suite = opts[:suite] || nil
  @stdout = opts[:quiet] ? nil : StdoutReport.new
  @list = []
  @compare = false
  @json_path = false
  @held_path = nil
  @held_results = nil

  @timing = {}
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1

  # Default statistical model
  @stats = :sd
  @confidence = 95
end

Instance Attribute Details

#compare ⇒ Boolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean) —

    true if needs to run compare.



19
20
21
 
# File 'lib/benchmark/ips/job.rb', line 19

def compare
  @compare
end

#confidence ⇒ Integer

Confidence.

Returns:

  • (Integer) 


51
52
53
 
# File 'lib/benchmark/ips/job.rb', line 51

def confidence
  @confidence
end

#full_report ⇒ Report (readonly)

Report object containing information about the run.

Returns:

  • (Report) —

    the report object.



27
28
29
 
# File 'lib/benchmark/ips/job.rb', line 27

def full_report
  @full_report
end

#hold ⇒ Boolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean) 


23
24
25
 
# File 'lib/benchmark/ips/job.rb', line 23

def hold
  @hold
end

#iterations ⇒ Integer

Warmup and calculation iterations.

Returns:

  • (Integer) 


43
44
45
 
# File 'lib/benchmark/ips/job.rb', line 43

def iterations
  @iterations
end

#list ⇒ Array<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>) —

    list of entries



15
16
17
 
# File 'lib/benchmark/ips/job.rb', line 15

def list
  @list
end

#stats ⇒ Object

Statistics model.

Returns:

  • (Object) 


47
48
49
 
# File 'lib/benchmark/ips/job.rb', line 47

def stats
  @stats
end

#time ⇒ Integer

Calculation time setter and getter (in seconds).

Returns:

  • (Integer) 


39
40
41
 
# File 'lib/benchmark/ips/job.rb', line 39

def time
  @time
end

#timing ⇒ Hash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash) 


31
32
33
 
# File 'lib/benchmark/ips/job.rb', line 31

def timing
  @timing
end

#warmup ⇒ Integer

Warmup time setter and getter (in seconds).

Returns:

  • (Integer) 


35
36
37
 
# File 'lib/benchmark/ips/job.rb', line 35

def warmup
  @warmup
end

Instance Method Details

#compare! ⇒ Object

Set @compare to true.



98
99
100
 
# File 'lib/benchmark/ips/job.rb', line 98

def compare!
  @compare = true
end

#compare? ⇒ Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean) —

    Need to compare?



93
94
95
 
# File 'lib/benchmark/ips/job.rb', line 93

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash) —

    a customizable set of options

  • iterations (Hash) —

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer) —

    Warmup time.

  • :time (Integer) —

    Calculation time.



82
83
84
85
86
87
88
89
 
# File 'lib/benchmark/ips/job.rb', line 82

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @suite = opts[:suite] if opts[:suite]
  @iterations = opts[:iterations] if opts[:iterations]
  @stats = opts[:stats] if opts[:stats]
  @confidence = opts[:confidence] if opts[:confidence]
end

#create_report(label, measured_us, iter, samples, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String) —

    Report item label.

  • measured_us (Integer) —

    Measured time in microsecond.

  • iter (Integer) —

    Iterations.

  • samples (Array<Float>) —

    Sampled iterations per second.

  • cycles (Integer) —

    Number of Cycles.

Returns:



344
345
346
 
# File 'lib/benchmark/ips/job.rb', line 344

def create_report(label, measured_us, iter, samples, cycles)
  @full_report.add_entry label, measured_us, iter, samples, cycles
end

#create_stats(samples) ⇒ Object 



316
317
318
319
320
321
322
323
324
325
 
# File 'lib/benchmark/ips/job.rb', line 316

def create_stats(samples)
  case @stats
    when :sd
      Stats::SD.new(samples)
    when :bootstrap
      Stats::Bootstrap.new(samples, @confidence)
    else
      raise "unknown stats #{@stats}"
  end
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float) —

    Each iteration’s time in ms.

  • iters (Integer) —

    Iterations.

Returns:

  • (Integer) —

    Cycles per 100ms.



148
149
150
151
 
# File 'lib/benchmark/ips/job.rb', line 148

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles <= 0 ? 1 : cycles
end

#generate_json ⇒ Object

Generate json from @full_report.



333
334
335
 
# File 'lib/benchmark/ips/job.rb', line 333

def generate_json
  @full_report.generate_json @json_path if json?
end

#held_results? ⇒ Boolean

Returns:

  • (Boolean) 


170
171
172
 
# File 'lib/benchmark/ips/job.rb', line 170

def held_results?
  File.exist?(@held_path)
end

#hold!(held_path) ⇒ Object

Set @hold to true.



109
110
111
 
# File 'lib/benchmark/ips/job.rb', line 109

def hold!(held_path)
  @held_path = held_path
end

#hold? ⇒ Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean) —

    Need to hold results between multiple Ruby invocations?



104
105
106
 
# File 'lib/benchmark/ips/job.rb', line 104

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "") —

    Label of benchmarked code.

  • str (String) (defaults to: nil) —

    Code to be benchmarked.

  • blk (Proc) —

    Code to be benchmarked.

Raises:

  • (ArgumentError) —

    Raises if str and blk are both present.

  • (ArgumentError) —

    Raises if str and blk are both absent.



130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/benchmark/ips/job.rb', line 130

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer) —

    Cycles.

  • time_us (Integer) —

    Time in microsecond.

Returns:

  • (Float) —

    Iteration per second.



166
167
168
 
# File 'lib/benchmark/ips/job.rb', line 166

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Set @json_path to given path, defaults to “data.json”.



120
121
122
 
# File 'lib/benchmark/ips/job.rb', line 120

def json!(path="data.json")
  @json_path = path
end

#json? ⇒ Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean) —

    Need to generate json?



115
116
117
 
# File 'lib/benchmark/ips/job.rb', line 115

def json?
  !!@json_path
end

#load_held_results ⇒ Object 



174
175
176
177
178
179
180
 
# File 'lib/benchmark/ips/job.rb', line 174

def load_held_results
  require "json"
  @held_results = Hash[File.open(@held_path).map { |line|
    result = JSON.parse(line)
    [result['item'], result]
  }]
end

#run ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/benchmark/ips/job.rb', line 182

def run
  @stdout.start_warming if @stdout
  @iterations.times do
    run_warmup
  end
  
  @stdout.start_running if @stdout
  
  held = nil
  
  @iterations.times do |n|
    held = run_benchmark
  end

  @stdout.footer if @stdout
  
  if held
    puts
    puts 'Pausing here -- run Ruby again to measure the next benchmark...'
  end
end

#run_benchmark ⇒ Object

Run calculation.



238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/benchmark/ips/job.rb', line 238

def run_benchmark
  @list.each do |item|
    if hold? && @held_results && @held_results.key?(item.label)
     result = @held_results[item.label]
      create_report(item.label, result['measured_us'], result['iter'],
                    create_stats(result['samples']), result['cycles'])
      next
    end
    
    @suite.running item.label, @time if @suite
    @stdout.running item.label, @time if @stdout

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time
    
    while (before = Timing.now) < target
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end

    final_time = before

    measured_us = measurements_us.inject(:+)

    samples = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @stdout.add_report rep, caller(1).first if @stdout
    @suite.add_report rep, caller(1).first if @suite
    
    if hold? && item != @list.last
      File.open @held_path, "a" do |f|
        require "json"
        f.write JSON.generate({
          :item => item.label,
          :measured_us => measured_us,
          :iter => iter,
          :samples => samples,
          :cycles => cycles
        })
        f.write "\n"
      end
      
      return true
    end
  end
  
  if hold? && @full_report.entries.size == @list.size
    File.delete @held_path if File.exist?(@held_path)
  end
  
  false
end

#run_comparison ⇒ Object

Run comparison of entries in @full_report.



328
329
330
 
# File 'lib/benchmark/ips/job.rb', line 328

def run_comparison
  @full_report.run_comparison if compare?
end

#run_warmup ⇒ Object

Run warmup.



205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
 
# File 'lib/benchmark/ips/job.rb', line 205

def run_warmup
  @list.each do |item|
    next if hold? && @held_results && @held_results.key?(item.label)
    
    @suite.warming item.label, @warmup if @suite
    @stdout.warming item.label, @warmup if @stdout

    Timing.clean_env

    before = Timing.now
    target = Timing.add_second before, @warmup

    warmup_iter = 0

    while Timing.now < target
      item.call_times(1)
      warmup_iter += 1
    end

    after = Timing.now

    warmup_time_us = Timing.time_us(before, after)

    @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter

    @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
    @suite.warmup_stats warmup_time_us, @timing[item] if @suite
    
    break if hold?
  end
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time) —

    time.

  • after (Time) —

    time.

Returns:

  • (Float) —

    Time difference of before and after.



157
158
159
 
# File 'lib/benchmark/ips/job.rb', line 157

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end