Class: OpenStudioMeasureTester::Runner

Inherits:
Object
  • Object
show all
Defined in:
lib/openstudio_measure_tester/runner.rb

Overview

The runner is the workhorse that executes the tests. This class does not invoke Rake and can be run as a library method or as a CLI call

Instance Method Summary collapse

Constructor Details

#initialize(base_dir) ⇒ Runner

Initialize the runner

Parameters:

  • base_dir (string) —

    Base dir, measures will be recursively tested from this location. Results will be here too.



15
16
17
18
19
 
# File 'lib/openstudio_measure_tester/runner.rb', line 15

def initialize(base_dir)
  @base_dir = base_dir

  puts "Base dir is #{base_dir}"
end

Instance Method Details

#dashboard ⇒ Object

Run ERB to create the dashboard



30
31
32
33
 
# File 'lib/openstudio_measure_tester/runner.rb', line 30

def dashboard
  template = OpenStudioMeasureTester::Dashboard.new(test_results_dir)
  template.render
end

#github_actions_report ⇒ Object 



35
36
37
38
39
 
# File 'lib/openstudio_measure_tester/runner.rb', line 35

def github_actions_report
  gha_reporter = OpenStudioMeasureTester::GithubActionsReport.new(test_results_dir)
  gha_reporter.make_minitest_step_summary_table
  gha_reporter.make_minitest_annotations
end

#post_process_results(original_results_directory = nil) ⇒ Object

Post process the various results and save them into the base_dir

Parameters:

  • original_results_directory (string) (defaults to: nil) —

    Location of the results from coverage and minitest



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/openstudio_measure_tester/runner.rb', line 96

def post_process_results(original_results_directory = nil)
  puts ' ========================= Starting Results Post Process ================================'
  puts "Current directory: #{@base_dir}"
  puts "Test results will be stored in: #{test_results_dir}"

  results = OpenStudioMeasureTester::OpenStudioTestingResult.new(@base_dir, test_results_dir, original_results_directory)
  results.save_results # one single file for dashboard

  # call the create dashboard command now that we have results
  dashboard

  github_actions_report if ENV['GITHUB_ACTIONS']

  # return the results exit code
  return results.exit_code
end

#pre_process_minitest(orig_results_dir) ⇒ Object

Prepare the current directory and the root directory to remove old test results before running the new tests

Parameters:

  • orig_results_dir (string) —

    Location on where there results of minitest/coverage will be stored.



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/openstudio_measure_tester/runner.rb', line 45

def pre_process_minitest(orig_results_dir)
  puts "Current directory is #{@base_dir}"
  puts "Pre-processed tests run data in #{orig_results_dir}"
  puts "Test results will be stored in: #{test_results_dir}"

  # There is a bunch of moving that needs to happen with coverage/minitest...
  FileUtils.rm_rf "#{orig_results_dir}/coverage" if Dir.exist? "#{orig_results_dir}/coverage"
  FileUtils.rm_rf "#{@base_dir}/coverage" if Dir.exist? "#{@base_dir}/coverage"
  FileUtils.rm_rf "#{test_results_dir}/coverage" if Dir.exist? "#{test_results_dir}/coverage"

  FileUtils.rm_rf "#{orig_results_dir}/minitest" if Dir.exist? "#{orig_results_dir}/minitest"
  FileUtils.rm_rf "#{@base_dir}/minitest" if Dir.exist? "#{@base_dir}/minitest"
  FileUtils.rm_rf "#{test_results_dir}/minitest" if Dir.exist? "#{test_results_dir}/minitest"

  FileUtils.rm_rf "#{orig_results_dir}/test" if Dir.exist? "#{orig_results_dir}/test"
  FileUtils.rm_rf "#{@base_dir}/test" if Dir.exist? "#{@base_dir}/test"
  # remove the test directory if it is empty (size == 2 for . and ..)
  if Dir.exist?("#{test_results_dir}/test") && Dir.entries("#{test_results_dir}/test").size == 2
    FileUtils.rm_rf "#{test_results_dir}/test"
  end
  FileUtils.rm_rf "#{test_results_dir}/minitest" if Dir.exist? "#{test_results_dir}/minitest"

  # Create the test_results directory to store all the results
  return test_results_dir
end

#pre_process_rubocop ⇒ Object

Rubocop stores the results (for now) in the test_results directory



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/openstudio_measure_tester/runner.rb', line 72

def pre_process_rubocop
  # copy over the .rubocop.yml file into the root directory of where this is running.
  shared_rubocop_file = File.expand_path('../../.rubocop.yml', File.dirname(__FILE__))
  dest_file = "#{File.expand_path(@base_dir)}/.rubocop.yml"
  if !File.exist?(dest_file)
    FileUtils.copy(shared_rubocop_file, dest_file)
  elsif File.read(shared_rubocop_file) != File.read(dest_file)
    puts "Using specific custom .rubocop.yml rather than the OpenStudio-measure-tester-gem's one"
  end

  puts "Current directory is #{@base_dir}"
  # puts "Pre-processing tests run in #{@base_dir}"
  puts "Test results will be stored in: #{test_results_dir}"

  FileUtils.rm_rf "#{test_results_dir}/rubocop" if Dir.exist? "#{test_results_dir}/rubocop"
  FileUtils.rm_rf "#{@base_dir}/rubocop" if Dir.exist? "#{@base_dir}/rubocop"

  # Call the test_results dir to create the directory and return it as a string.
  return test_results_dir
end

#pre_process_style ⇒ Object

OpenStudio style check preparation

Parameters:

  • base_dir (string) —

    Base directory where results will be stored. If called from rake it is the location of the Rakefile.

  • measures_dir (string) —

    The current working directory. If called from Rake it is the active directory.



117
118
119
120
121
122
123
124
125
126
 
# File 'lib/openstudio_measure_tester/runner.rb', line 117

def pre_process_style
  puts "Current directory is #{@base_dir}"
  puts "Test results will be stored in: #{test_results_dir}"

  FileUtils.rm_rf "#{test_results_dir}/openstudio_style" if Dir.exist? "#{test_results_dir}/openstudio_style"
  FileUtils.rm_rf "#{@base_dir}/openstudio_style" if Dir.exist? "#{@base_dir}/openstudio_style"

  # Call the test_results dir to create the directory and return it as a string.
  return test_results_dir
end

#run_all(original_results_directory) ⇒ Object 



345
346
347
348
349
350
351
 
# File 'lib/openstudio_measure_tester/runner.rb', line 345

def run_all(original_results_directory)
  # do not run coverage now since the at_exit is causing exceptions when running (GC?)
  run_rubocop(true)
  run_style(true)
  run_test(true, original_results_directory, false)
  post_process_results(original_results_directory)
end

#run_rubocop(skip_post_process, auto_correct = false) ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/openstudio_measure_tester/runner.rb', line 143

def run_rubocop(skip_post_process, auto_correct = false)
  puts ' ========================= Starting Run for Rubocop ================================'
  pre_process_rubocop

  rubocop_results_file = "#{test_results_dir}/rubocop/rubocop-results.xml"
  # The documentation on running RuboCop from the Ruby source is not really helpful. I reversed engineered this
  # by putting in puts statements in my local install rubocop gem to see how options were passed.
  #
  # https://github.com/bbatsov/rubocop/blob/9bdbaba9dcaa3dedad5e857b440d0d8988b806f5/lib/rubocop/runner.rb#L25
  require 'rubocop/formatter/checkstyle_formatter'
  options = {
    # out and output_path do not actually save the results, has to be appended after the formatter.
    # out: 'junk.xml',
    # output_path: 'junk.xml',
    auto_correct:,
    color: false,
    # cf #76 - Because we pass a glob to the Runner.run, we must pass
    # force_exclusion to respect the files excluded in the .rubocop.yml
    force_exclusion: true,
    formatters: ['simple', ['RuboCop::Formatter::CheckstyleFormatter', rubocop_results_file]]
  }

  # Load in the ruby config from the root directory
  config_store = RuboCop::ConfigStore.new
  # puts "Searching for .rubocop.yml recursively from #{@base_dir}"
  config_store.for(@base_dir)

  rc_runner = RuboCop::Runner.new(options, config_store)
  rc_runner.run(["#{File.expand_path(@base_dir)}/**/*.rb"])

  if skip_post_process
    return true
  else
    return post_process_results
  end
end

#run_style(skip_post_process) ⇒ Object 



128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/openstudio_measure_tester/runner.rb', line 128

def run_style(skip_post_process)
  puts ' ========================= Starting Run for OpenStudio Style ================================'
  pre_process_style

  # Run the style tests
  style = OpenStudioMeasureTester::OpenStudioStyle.new(test_results_dir, "#{@base_dir}/**/measure.rb")
  style.save_results

  if skip_post_process
    return true
  else
    return post_process_results
  end
end

#run_test(skip_post_process, original_results_directory, run_coverage = true) ⇒ Object

The results of the coverage and minitest are stored in the root of the directory structure (if Rake)



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
 
# File 'lib/openstudio_measure_tester/runner.rb', line 181

def run_test(skip_post_process, original_results_directory, run_coverage = true)
  puts ' ========================= Starting Run for Minitest (and coverage) ============================'
  # not sure what @base_dir has to be right now
  pre_process_minitest(original_results_directory)

  # Specify the minitest reporters
  require 'minitest/reporters'
  Minitest::Reporters.use! [
    Minitest::Reporters::HtmlReporter.new,
    Minitest::Reporters::JUnitReporter.new
  ]

  if run_coverage
    # Load in the coverage before loading the test files
    SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new(
      [
        SimpleCov::Formatter::HTMLFormatter
      ]
    )

    SimpleCov.start do
      # Track all files inside of @base_dir
      track_files "#{@base_dir}/**/*.rb"

      use_merging false

      # Exclude all files outside of @base_dir
      root_filter = nil
      add_filter do |src|
        root_filter ||= /\A#{Regexp.escape(@base_dir + File::SEPARATOR)}/io
        src.filename !~ root_filter
      end
    end
  end

  num_tests = 0
  openstudio_version = OpenStudio::VersionString.new(OpenStudio.openStudioVersion)
  Dir["#{@base_dir}/**/*_Test.rb", "#{@base_dir}/**/*_test.rb"].uniq.each do |file|
    file = File.expand_path(file)
    measure_dir = File.expand_path(File.join(File.dirname(file), '..'))

    # check measure xml
    compatible = {
      compatible: true,
      message: '',
      openstudio_version: openstudio_version.str,
      measure_min_version: 'None',
      measure_max_version: 'None',
      loaded: false,
      load_errors: []
    }
    begin
      measure = OpenStudio::BCLMeasure.new(measure_dir)
      compatible[:measure_name] = measure.className
      measure.files.each do |f|
        if f.fileName == 'measure.rb'
          if !f.minCompatibleVersion.empty?
            min_version = f.minCompatibleVersion.get
            compatible[:measure_min_version] = min_version.str
            if openstudio_version < min_version
              compatible[:compatible] = false
              compatible[:message] = "OpenStudio Version #{openstudio_version.str} < Min Version #{min_version.str}"
            end
          end
          if !f.maxCompatibleVersion.empty?
            max_version = f.maxCompatibleVersion.get
            compatible[:measure_max_version] = max_version.str
            if openstudio_version > max_version
              compatible[:compatible] = false
              compatible[:message] = "OpenStudio Version #{openstudio_version.str} > Max Version #{max_version.str}"
            end
          end
        end
      end
    rescue StandardError => e
      compatible[:compatible] = false
      compatible[:message] = e.message
    end

    if !compatible[:compatible]
      puts "Measure not compatible: #{measure_dir}, #{compatible[:message]}"
      next
    end

    # load test
    puts "Loading file for testing: #{file}"
    begin
      load file
      compatible[:loaded] = true
      num_tests += 1
    rescue StandardError, LoadError => e
      compatible[:load_errors] << e.message
    end

  ensure
    # Write out the compatibility
    # write out to a file that the measure is not applicable
    os_compatible_file = "#{@base_dir}/test_results/minitest/compatibility/#{compatible[:measure_name]}.json"
    puts os_compatible_file
    FileUtils.mkdir_p File.dirname(os_compatible_file) unless Dir.exist? File.dirname(os_compatible_file)
    File.open(os_compatible_file, 'w') do |f|
      f << JSON.pretty_generate(compatible)
    end
  end

  if num_tests < 1
    puts 'No tests found'

    if run_coverage
      # This doesn't seem to be working, it doesn't save off the .resultset.json.
      begin
        simplecov_exit_status = SimpleCov.end_now
      rescue NoMethodError
        # in case using some other version of SimpleCov
        SimpleCov.set_exit_exception
        exit_status = SimpleCov.exit_status_from_exception
        SimpleCov.result.format!
        simplecov_exit_status = SimpleCov.process_result(SimpleCov.result, exit_status)
      end
    end

    if skip_post_process
      return true
    else
      return post_process_results(original_results_directory)
    end
  else
    puts "Inspected #{num_tests} tests"
  end

  # Now call run on the loaded files. Note that the Minitest.autorun method has been nulled out in the
  # openstudio_measure_tester.rb file, so it will not run.
  begin
    Minitest.run ['--verbose']
  rescue StandardError => e
    puts
    puts '!!!!!!!!!!!!!!!!!!!!! Minitest Error Occurred !!!!!!!!!!!!!!!!!!!!!'
    puts e.message
    puts e.backtrace
    puts '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
    puts
  end

  # Shutdown SimpleCov and collect results
  # This will set SimpleCov.running to false which will prevent from running again at_exit
  if run_coverage
    begin
      simplecov_exit_status = SimpleCov.end_now
    rescue NoMethodError
      # in case using some other version of SimpleCov
      SimpleCov.set_exit_exception
      exit_status = SimpleCov.exit_status_from_exception
      SimpleCov.result.format!
      simplecov_exit_status = SimpleCov.process_result(SimpleCov.result, exit_status)
    end
  end

  if skip_post_process
    return true
  else
    return post_process_results(original_results_directory)
  end
end

#test_results_dir ⇒ Object

Create and return the location where the tests results need to be stored



22
23
24
25
26
27
 
# File 'lib/openstudio_measure_tester/runner.rb', line 22

def test_results_dir
  dir = "#{@base_dir}/test_results"
  FileUtils.mkdir_p dir unless Dir.exist? dir

  return dir
end