Class: Gravitext::PerfTest::Harness

Inherits:
Object
  • Object
show all
Includes:
CalcUtil
Defined in:
lib/gravitext-util/perftest.rb

Overview

Concurrent performance testing harness with support for adaptive warmup and comparison on multiple TestFactory instances.

Constant Summary

Constants included from CalcUtil

CalcUtil::NaN

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from CalcUtil

#latency_change, #throughput_change

Constructor Details

#initialize(factories) ⇒ Harness

Initialize given array of com.gravitext.concurrent.TestFactory instances.



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/gravitext-util/perftest.rb', line 92

def initialize( factories )
  @factories = factories
  @thread_count = Concurrent::available_cores

  @warmup_exec_target = 8.0
  @warmup_total_target = 25
  @warmup_tolerance = 0.05

  @final_iterations = 3
  @final_runs = nil
  @final_exec_target = 10.0

  @max_align_ratio = 3.0
  @do_per_run_timing = true

  @listener = PrintListener.new
end

Instance Attribute Details

#do_per_run_timing ⇒ Object

Do per run timing for mean latency. (default: true)



84
85
86
 
# File 'lib/gravitext-util/perftest.rb', line 84

def do_per_run_timing
  @do_per_run_timing
end

#final_exec_target ⇒ Object

Target duration of final comparison iterations in seconds (default: 10s)



78
79
80
 
# File 'lib/gravitext-util/perftest.rb', line 78

def final_exec_target
  @final_exec_target
end

#final_iterations ⇒ Object

Number of final comparison iterations (default: 3)



71
72
73
 
# File 'lib/gravitext-util/perftest.rb', line 71

def final_iterations
  @final_iterations
end

#final_runs ⇒ Object

Number of runs in final comparisons (default: computed from warmup and final_exec_target)



75
76
77
 
# File 'lib/gravitext-util/perftest.rb', line 75

def final_runs
  @final_runs
end

#listener ⇒ Object

Test executation progress is sent to listener, see PrintListener.



81
82
83
 
# File 'lib/gravitext-util/perftest.rb', line 81

def listener
  @listener
end

#max_align_ratio ⇒ Object

Maximum difference in counts for final run counts to be alligned (default: 3.0)



88
89
90
 
# File 'lib/gravitext-util/perftest.rb', line 88

def max_align_ratio
  @max_align_ratio
end

#thread_count ⇒ Object

Number of threads to test with (default: available cores)



58
59
60
 
# File 'lib/gravitext-util/perftest.rb', line 58

def thread_count
  @thread_count
end

#warmup_exec_target ⇒ Object

Target duration of warmup comparison iterations in seconds (default:8s)



61
62
63
 
# File 'lib/gravitext-util/perftest.rb', line 61

def warmup_exec_target
  @warmup_exec_target
end

#warmup_tolerance ⇒ Object

Secondary warmup requirement: consectutive iteration throughputs within tollerance (default: 0.05)



68
69
70
 
# File 'lib/gravitext-util/perftest.rb', line 68

def warmup_tolerance
  @warmup_tolerance
end

#warmup_total_target ⇒ Object

Minimum warmup time in seconds for each test factory (default: 25s)



64
65
66
 
# File 'lib/gravitext-util/perftest.rb', line 64

def warmup_total_target
  @warmup_total_target
end

Instance Method Details

#align_counts(counts) ⇒ Object 



185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/gravitext-util/perftest.rb', line 185

def align_counts( counts )
  mean = ( counts.inject { |sum,c| sum + c } ) / counts.size

  # Round to 2-significant digits
  f = 1
  ( f *= 10 ) while ( mean / f ) > 100
  mean = ( mean.to_f / f ).round * f

  if ( mean.to_f / counts.min ) > @max_align_ratio
    counts
  else
    Array.new( counts.size, mean )
  end
end

#create_executor(factory, runs) ⇒ Object 



256
257
258
259
260
 
# File 'lib/gravitext-util/perftest.rb', line 256

def create_executor( factory, runs )
  executor = TestExecutor.new( factory, runs, @thread_count )
  executor.do_per_run_timing = @do_per_run_timing
  executor
end

#execute ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/gravitext-util/perftest.rb', line 110

def execute
  @listener.begin( self, @factories )

  finals = warmup

  run_counts = if @final_runs
                 Array.new( @factories.size, @final_runs )
               else
                 counts = @factories.map do |factory|
                   exec = finals.detect { |e| e.factory == factory }
                   [ ( exec.mean_throughput * @final_exec_target ).to_i,
                     1 ].max
                 end
                 align_counts( counts )
               end

  results = execute_comparisons( run_counts, @final_iterations )
  sums = sum_results( results ) if @final_iterations > 1

  @listener.comparisons_end( self, sums )
  sums
end

#execute_comparisons(run_counts, iterations) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
 
# File 'lib/gravitext-util/perftest.rb', line 200

def execute_comparisons( run_counts, iterations )

  results = Array.new( @factories.size ) { [] }

  @listener.comparisons_begin( self, run_counts )

  iterations.times do |iteration|
    @listener.comparison_next_series( self ) unless iteration.zero?

    Java::java.lang.System::gc
    Java::java.lang.Thread::yield

    @factories.each_index do |f|
      executor = create_executor( @factories[f], run_counts[f] )
      @listener.comparison_start_run( executor )
      executor.run_test
      @listener.comparison_complete_run( executor, results[0].last )
      results[f] << executor
    end
  end

  results
end

#sum_results(results) ⇒ Object 



224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
 
# File 'lib/gravitext-util/perftest.rb', line 224

def sum_results( results )
  sums = []

  results.each do |runs|
    sum = OpenStruct.new
    sum.factory            = runs.first.factory
    sum.runs_target        = 0
    sum.duration           = 0.0
    sum.result_sum         = 0
    sum.runs_executed      = 0
    sum.mean_throughput    = 0.0
    sum.mean_latency       = 0.0

    runs.each do |exec|
      sum.runs_target     += exec.runs_target
      sum.duration        += exec.duration.seconds
      sum.result_sum      += exec.result_sum
      sum.runs_executed   += exec.runs_executed
      sum.mean_throughput += exec.mean_throughput
      sum.mean_latency    += exec.mean_latency.seconds
    end

    sum.duration           = Duration.new( sum.duration )
    sum.mean_throughput   /= runs.size
    sum.mean_latency       = Duration.new( sum.mean_latency / runs.size )

    sums << sum
  end

  sums
end

#warmup ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/gravitext-util/perftest.rb', line 133

def warmup

  @listener.warmups_begin( self )

  states = @factories.map do |factory|
    s = OpenStruct.new
    s.factory = factory
    s.prior = nil
    s.warm_time = 0.0
    s
  end

  first = true
  finals = []
  until states.empty? do

    @listener.warmup_next_series( self ) unless first

    states,done = states.partition do |s|

      # Cleanup before each run
      Java::java.lang.System::gc
      Java::java.lang.Thread::yield

      runs = if s.prior
               [ ( @warmup_exec_target * s.prior.runs_executed ) /
                 s.prior.duration.seconds, 1 ].max
             else
               1
             end

      executor = create_executor( s.factory, runs.to_i )
      @listener.warmup_start_run( executor )
      executor.run_test
      @listener.warmup_complete_run( executor, s.prior )

      # Test throughput change, and increment warm_time
      s.warm_time += executor.duration.seconds
      tchange = throughput_change( executor, s.prior )
      s.prior = executor

      ( ( s.warm_time < @warmup_total_target ) ||
        ( tchange.abs > @warmup_tolerance ) )
    end
    finals += done.map { |s| s.prior }
    first = false
  end

  @listener.warmups_end( finals )
  finals
end