Class: Benchmark::IPS::Job

Inherits:
Object
  • Object
show all
Defined in:
lib/benchmark/ips/job.rb,
lib/benchmark/ips/job/entry.rb,
lib/benchmark/ips/job/stdout_report.rb

Overview

Benchmark jobs.

Defined Under Namespace

Classes: Entry, StdoutReport

Constant Summary collapse

MICROSECONDS_PER_100MS =

Microseconds per 100 millisecond.

100_000
MICROSECONDS_PER_SECOND =

Microseconds per second.

Timing::MICROSECONDS_PER_SECOND
MAX_TIME_SKEW =

The percentage of the expected runtime to allow before reporting a weird runtime

0.05

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(opts = {}) ⇒ Job

Instantiate the Benchmark::IPS::Job.

Parameters:

  • opts (Hash) (defaults to: {}) —

    a customizable set of options

Options Hash (opts):

  • (nil) (Benchmark::Suite) —

    :suite Specify Benchmark::Suite.

  • (false) (Boolean) —

    :quiet Suppress the printing of information.



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/benchmark/ips/job.rb', line 48

def initialize opts={}
  @suite = opts[:suite] || nil
  @stdout = opts[:quiet] ? nil : StdoutReport.new
  @list = []
  @compare = false
  @json_path = false
  @held_path = nil
  @held_results = nil

  @timing = {}
  @full_report = Report.new

  # Default warmup and calculation time in seconds.
  @warmup = 2
  @time = 5
  @iterations = 1
end

Instance Attribute Details

#compare ⇒ Boolean (readonly)

Determining whether to run comparison utility.

Returns:

  • (Boolean) —

    true if needs to run compare.



19
20
21
 
# File 'lib/benchmark/ips/job.rb', line 19

def compare
  @compare
end

#full_report ⇒ Report (readonly)

Report object containing information about the run.

Returns:

  • (Report) —

    the report object.



27
28
29
 
# File 'lib/benchmark/ips/job.rb', line 27

def full_report
  @full_report
end

#hold ⇒ Boolean

Determining whether to hold results between Ruby invocations

Returns:

  • (Boolean) 


23
24
25
 
# File 'lib/benchmark/ips/job.rb', line 23

def hold
  @hold
end

#iterations ⇒ Integer

Warmup and calculation iterations.

Returns:

  • (Integer) 


43
44
45
 
# File 'lib/benchmark/ips/job.rb', line 43

def iterations
  @iterations
end

#list ⇒ Array<Entry> (readonly)

Two-element arrays, consisting of label and block pairs.

Returns:

  • (Array<Entry>) —

    list of entries



15
16
17
 
# File 'lib/benchmark/ips/job.rb', line 15

def list
  @list
end

#time ⇒ Integer

Calculation time setter and getter (in seconds).

Returns:

  • (Integer) 


39
40
41
 
# File 'lib/benchmark/ips/job.rb', line 39

def time
  @time
end

#timing ⇒ Hash (readonly)

Storing Iterations in time period.

Returns:

  • (Hash) 


31
32
33
 
# File 'lib/benchmark/ips/job.rb', line 31

def timing
  @timing
end

#warmup ⇒ Integer

Warmup time setter and getter (in seconds).

Returns:

  • (Integer) 


35
36
37
 
# File 'lib/benchmark/ips/job.rb', line 35

def warmup
  @warmup
end

Instance Method Details

#compare! ⇒ Object

Set @compare to true.



84
85
86
 
# File 'lib/benchmark/ips/job.rb', line 84

def compare!
  @compare = true
end

#compare? ⇒ Boolean

Return true if job needs to be compared.

Returns:

  • (Boolean) —

    Need to compare?



79
80
81
 
# File 'lib/benchmark/ips/job.rb', line 79

def compare?
  @compare
end

#config(opts) ⇒ Object

Job configuration options, set @warmup and @time.

Parameters:

  • opts (Hash) —

    a customizable set of options

  • iterations (Hash) —

    a customizable set of options

Options Hash (opts):

  • :warmup (Integer) —

    Warmup time.

  • :time (Integer) —

    Calculation time.



70
71
72
73
74
75
 
# File 'lib/benchmark/ips/job.rb', line 70

def config opts
  @warmup = opts[:warmup] if opts[:warmup]
  @time = opts[:time] if opts[:time]
  @suite = opts[:suite] if opts[:suite]
  @iterations = opts[:iterations] if opts[:iterations]
end

#create_report(label, measured_us, iter, avg_ips, sd_ips, cycles) ⇒ Report::Entry

Create report by add entry to @full_report.

Parameters:

  • label (String) —

    Report item label.

  • measured_us (Integer) —

    Measured time in microsecond.

  • iter (Integer) —

    Iterations.

  • avg_ips (Float) —

    Average iterations per second.

  • sd_ips (Float) —

    Standard deviation iterations per second.

  • cycles (Integer) —

    Number of Cycles.

Returns:



323
324
325
 
# File 'lib/benchmark/ips/job.rb', line 323

def create_report(label, measured_us, iter, avg_ips, sd_ips, cycles)
  @full_report.add_entry label, measured_us, iter, avg_ips, sd_ips, cycles
end

#cycles_per_100ms(time_msec, iters) ⇒ Integer

Calculate the cycles needed to run for approx 100ms, given the number of iterations to run the given time.

Parameters:

  • time_msec (Float) —

    Each iteration’s time in ms.

  • iters (Integer) —

    Iterations.

Returns:

  • (Integer) —

    Cycles per 100ms.



134
135
136
137
138
 
# File 'lib/benchmark/ips/job.rb', line 134

def cycles_per_100ms time_msec, iters
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
  cycles = 1 if cycles <= 0
  cycles
end

#generate_json ⇒ Object

Generate json from @full_report.



311
312
313
 
# File 'lib/benchmark/ips/job.rb', line 311

def generate_json
  @full_report.generate_json @json_path if json?
end

#held_results? ⇒ Boolean

Returns:

  • (Boolean) 


157
158
159
 
# File 'lib/benchmark/ips/job.rb', line 157

def held_results?
  File.exist?(@held_path)
end

#hold!(held_path) ⇒ Object

Set @hold to true.



95
96
97
 
# File 'lib/benchmark/ips/job.rb', line 95

def hold!(held_path)
  @held_path = held_path
end

#hold? ⇒ Boolean

Return true if results are held while multiple Ruby invocations

Returns:

  • (Boolean) —

    Need to hold results between multiple Ruby invocations?



90
91
92
 
# File 'lib/benchmark/ips/job.rb', line 90

def hold?
  !!@held_path
end

#item(label = "", str = nil, &blk) ⇒ Object Also known as: report

Registers the given label and block pair in the job list.

Parameters:

  • label (String) (defaults to: "") —

    Label of benchmarked code.

  • str (String) (defaults to: nil) —

    Code to be benchmarked.

  • blk (Proc) —

    Code to be benchmarked.

Raises:

  • (ArgumentError) —

    Raises if str and blk are both present.

  • (ArgumentError) —

    Raises if str and blk are both absent.



116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/benchmark/ips/job.rb', line 116

def item(label="", str=nil, &blk) # :yield:
  if blk and str
    raise ArgumentError, "specify a block and a str, but not both"
  end

  action = str || blk
  raise ArgumentError, "no block or string" unless action

  @list.push Entry.new(label, action)
  self
end

#iterations_per_sec(cycles, time_us) ⇒ Float

Calculate the interations per second given the number of cycles run and the time in microseconds that elapsed.

Parameters:

  • cycles (Integer) —

    Cycles.

  • time_us (Integer) —

    Time in microsecond.

Returns:

  • (Float) —

    Iteration per second.



153
154
155
 
# File 'lib/benchmark/ips/job.rb', line 153

def iterations_per_sec cycles, time_us
  MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
end

#json!(path = "data.json") ⇒ Object

Set @json_path to given path, defaults to “data.json”.



106
107
108
 
# File 'lib/benchmark/ips/job.rb', line 106

def json!(path="data.json")
  @json_path = path
end

#json? ⇒ Boolean

Return true if job needs to generate json.

Returns:

  • (Boolean) —

    Need to generate json?



101
102
103
 
# File 'lib/benchmark/ips/job.rb', line 101

def json?
  !!@json_path
end

#load_held_results ⇒ Object 



161
162
163
164
165
166
167
 
# File 'lib/benchmark/ips/job.rb', line 161

def load_held_results
  require "json"
  @held_results = Hash[File.open(@held_path).map { |line|
    result = JSON.parse(line)
    [result['item'], result]
  }]
end

#run ⇒ Object 



169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
 
# File 'lib/benchmark/ips/job.rb', line 169

def run
  @stdout.start_warming if @stdout
  @iterations.times do
    run_warmup
  end
  
  @stdout.start_running if @stdout
  
  held = nil
  
  @iterations.times do |n|
    held = run_benchmark
  end
  
  if held
    puts
    puts 'Pausing here -- run Ruby again to measure the next benchmark...'
  end
end

#run_benchmark ⇒ Object

Run calculation.



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/benchmark/ips/job.rb', line 223

def run_benchmark
  @list.each do |item|
    if hold? && @held_results && @held_results.key?(item.label)
     result = @held_results[item.label]
      create_report(item.label, result['measured_us'], result['iter'],
        result['avg_ips'], result['sd_ips'], result['cycles'])
      next
    end
    
    @suite.running item.label, @time if @suite
    @stdout.running item.label, @time if @stdout

    Timing.clean_env

    iter = 0

    measurements_us = []

    # Running this number of cycles should take around 100ms.
    cycles = @timing[item]

    target = Timing.add_second Timing.now, @time
    
    while (before = Timing.now) < target
      item.call_times cycles
      after = Timing.now

      # If for some reason the timing said this took no time (O_o)
      # then ignore the iteration entirely and start another.
      iter_us = Timing.time_us before, after
      next if iter_us <= 0.0

      iter += cycles

      measurements_us << iter_us
    end

    final_time = before

    measured_us = measurements_us.inject(0) { |a,i| a + i }

    all_ips = measurements_us.map { |time_us|
      iterations_per_sec cycles, time_us
    }

    avg_ips = Timing.mean(all_ips)
    sd_ips =  Timing.stddev(all_ips, avg_ips).round

    rep = create_report(item.label, measured_us, iter, avg_ips, sd_ips, cycles)

    if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
      rep.show_total_time!
    end

    @stdout.add_report rep, caller(1).first if @stdout
    @suite.add_report rep, caller(1).first if @suite
    
    if hold? && item != @list.last
      File.open @held_path, "a" do |f|
        require "json"
        f.write JSON.generate({
          :item => item.label,
          :measured_us => measured_us,
          :iter => iter,
          :avg_ips => avg_ips,
          :sd_ips => sd_ips,
          :cycles => cycles
        })
        f.write "\n"
      end
      
      return true
    end
  end
  
  if hold? && @full_report.entries.size == @list.size
    File.delete @held_path if File.exist?(@held_path)
  end
  
  false
end

#run_comparison ⇒ Object

Run comparison of entries in @full_report.



306
307
308
 
# File 'lib/benchmark/ips/job.rb', line 306

def run_comparison
  @full_report.run_comparison if compare?
end

#run_warmup ⇒ Object

Run warmup.



190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
 
# File 'lib/benchmark/ips/job.rb', line 190

def run_warmup
  @list.each do |item|
    next if hold? && @held_results && @held_results.key?(item.label)
    
    @suite.warming item.label, @warmup if @suite
    @stdout.warming item.label, @warmup if @stdout

    Timing.clean_env

    before = Timing.now
    target = Timing.add_second before, @warmup

    warmup_iter = 0

    while Timing.now < target
      item.call_times(1)
      warmup_iter += 1
    end

    after = Timing.now

    warmup_time_us = Timing.time_us(before, after)

    @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter

    @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
    @suite.warmup_stats warmup_time_us, @timing[item] if @suite
    
    break if hold?
  end
end

#time_us(before, after) ⇒ Float

Calculate the time difference of before and after in microseconds.

Parameters:

  • before (Time) —

    time.

  • after (Time) —

    time.

Returns:

  • (Float) —

    Time difference of before and after.



144
145
146
 
# File 'lib/benchmark/ips/job.rb', line 144

def time_us before, after
  (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
end