Module: CodeRunner::Gs2::Astrogk::TestAstrogk

Defined in:
lib/gs2crmod/astrogk/test_gs2.rb

Overview

Astrogk Test Suite

This class is designed to run a set of functional tests to test the GS2 source. It is not a test suite for GS2crmod, the GS2 CodeRunner module. It takes all the input files from the folder test_cases, runs them and checks that they give the same output as the standard case. Only the NetCDF file from the standard case should be included, and should have the same name as the test case input file.

Constant Summary collapse

TEST_FOLDER =

The folder where all the test files are stored.

Dir.pwd
TEST_OUT =

The io object all test info is written to.

STDERR
TOLERANCE =

Relative error which differences cannot exceed in the standard case

1.0e-10
EXCLUDED_VARIABLES =

A list of variables that are allowed to be different in the standard test

['input_file']

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.results_folder(input_file) ⇒ Object

Give the standardised name of the test_results folder, given the name of the test case input file



23
24
25
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 23

def self.results_folder(input_file)
  'test_results/' + input_file.sub(/\.in$/, '')
end

.run_checks(run) ⇒ Object 



153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 153

def self.run_checks(run)
  checks_failed = []
  
  input_text = File.read("#{test_case_folder(run.run_name)}/#{run.run_name}.in")
  description = input_text.scan(Regexp.new("#{/^\s*!\s*description\s*=\s* /}(#{Regexp.quoted_string})")).flatten[0]
  description.gsub!(/(.{25,45} |.{45})/){"#$1\n"} if description
  if input_text =~ Regexp.new("#{/^\s*!\s*custom_checks\s*=\s* /}(#{Regexp.properly_nested("\\[", "\\]", false)})")
    custom_checks = eval($1)
    custom_checks.each{|check| checks_failed.push check unless run.run_check check}
  else
    checks_failed.push :standard unless run.run_check :standard
  end
  unless checks_failed.size == 0
    @tests_failed[run.run_name] = {checks_failed: checks_failed, description: description}
  end
end

.run_tests ⇒ Object

Check to see which run has completed and run tests for those runs.



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 97

def self.run_tests
  eputs "Waiting for runs to complete"
  loop do
      break if @submitted_tests.size ==0
      @test_runner.update(false) 
      i = 0
      loop do
        tst = @submitted_tests[0]
        run = @test_runner.runs.find{|run| run.run_name == tst}
        if [:Complete, :Failed].include? run.status 
          run_checks run
          @submitted_tests.delete(tst)
        end
        i += 1
        break if i >= @submitted_tests.size
      end
      sleep 3
    end
end

.submit_run(test_case) ⇒ Object

Run the test case with the given input file.



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 120

def self.submit_run(test_case)
  
  dir = results_folder(test_case)
  input_text = File.read("#{test_case_folder(test_case)}/#{test_case}.in")
  
  description = input_text.scan(Regexp.new("#{/^\s*!\s*description\s*=\s* /}(#{Regexp.quoted_string})")).flatten[0]
  (eputs "----Rejecting '#{file}', no description provided or description is not in the correct format: description = \" description \"";return) unless description
  run = Astrogk.new(@test_runner)

  # This cryptic statement updates the run parameters from the input file 
  run.instance_eval(
    Astrogk.defaults_file_text_from_input_file("#{test_case_folder(test_case)}/#{test_case}.in"))
  
  run.run_name = test_case
  run.instance_variable_set(:@dir_name, dir)
  @test_runner.test_submission = true
  @test_runner.submit(run)
  
  other_files = Dir.entries(test_case_folder(test_case)).find_all do |f|
    not (f =~ /\.in/ or f =~ /\.out\.nc/ or f =~ /\.svn/ or [".", ".."].include? f)
  end
  other_files.each do |f|
#           p f
    FileUtils.cp(test_case_folder(test_case)+'/'+f, results_folder(test_case)+'/'+f)
  end
  
  @test_runner.test_submission = false
  @test_runner.submit(run)
  @submitted_tests.push run.run_name
  
  
end

.test_case_folder(input_file) ⇒ Object 



27
28
29
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 27

def self.test_case_folder(input_file)
  TEST_FOLDER + '/' + input_file.sub(/\.in$/, '') 
end

.test_gs2(test_case_location, options = {}) ⇒ Object

Run all the test cases. test_case_location should be the folder test_cases in the gs2 source.

Options are

  • restart (default: true) Delete all results and start again.

  • submit (default: true) Submit any runs that haven’t been submitted.

  • serial (default: true) Wait till one test run has finished before starting another



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 42

def self.test_gs2(test_case_location, options={})
    Astrogk.send(:include, self)
    test_case_location.sub!(/~/, ENV['HOME'])
    TEST_FOLDER.gsub!(/^.*$/, test_case_location)
    raise "The first argument should be the test_cases folder" unless File.basename(TEST_FOLDER) == "test_cases"
    @tests_failed = {}

    
    serial = true unless ["false", false].include? options[:serial]
    restart = true unless ["false", false].include? options[:restart]
    submit = true unless ["false", false].include? options[:submit]
    
    
    @test_runner = CodeRunner.fetch_runner(Y: Dir.pwd, u: true)
    @submitted_tests = @test_runner.runs.map{|run| run.run_name}
    test_cases = Dir.entries(TEST_FOLDER).find_all do |entry|
#             p entry
      File.directory?(TEST_FOLDER + '/' + entry) and not entry =~ /^\./
    end
#           p test_cases
    
    if restart
      FileUtils.rm_r 'test_results' if FileTest.exist? 'test_results' 
      FileUtils.makedirs 'test_results'
    end
    
    # Submit the tests
    if submit
      test_cases.each do |test_case|
        next if FileTest.exist? results_folder(test_case)
        submit_run(test_case)
        run_tests if serial
      end
    end
    run_tests
    
    if @tests_failed.size == 0
      TEST_OUT.puts "All Tests Completed Successfully"
      eputs "All Tests Completed Successfully" unless TEST_OUT == STDERR
    else
      eputs "Tests Were Failed" 
      @tests_failed.each do |name, hash|
        sep = "----------------------------------------------"
        TEST_OUT.puts '', sep, "                 Test Failed", sep
        TEST_OUT.puts "Name: #{name}"
        TEST_OUT.puts "Description: \n#{hash[:description]}", ''
        TEST_OUT.puts "This test failed on these checks: #{hash[:checks_failed]}", ''
        TEST_OUT.puts sep, ''
      end
    end
          
end

Instance Method Details

#run_check(check) ⇒ Object

Check the results against standard cases using the checks described here. Any custom tests should be implemented here!



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/gs2crmod/astrogk/test_gs2.rb', line 180

def run_check(check)
  netcdf_standard_case = NumRu::NetCDF.open(TestAstrogk.test_case_folder(@run_name) + '/' + @run_name + '.out.nc')
  Dir.chdir(@directory) do
    case check
    when :standard
      netcdf = NumRu::NetCDF.open(@run_name + '.out.nc')
      netcdf.vars.map{|v| v.name}.each do |v|
        unless netcdf_standard_case.var(v)
          TEST_OUT.puts "Warning: variable '#{v}' is missing from the test case netcdf output for '#@run_name'. Suggest updating the test case netcdf file. This is not a GS2 fault."
        end
      end
      netcdf_standard_case.vars.map{|v| v.name}.each do |v|
        next if EXCLUDED_VARIABLES.include? v
        begin
          unless netcdf.var(v)
            TEST_OUT.puts "Error: Variable #{v} is missing from the netcdf output for #@run_name"
            return false
          end
          
          narray = netcdf.var(v).get
          standard_narray = netcdf_standard_case.var(v).get
          
          if standard_narray.abs.max < TOLERANCE 
            if narray.abs.max < TOLERANCE 
              next
            else
              TEST_OUT.puts "Error: Variable '#{v}' has failed check in '#@run_name'"
              return false
            end
          end
          
          
          difference = narray - standard_narray
          return false unless difference.abs.max / standard_narray.abs.max < TOLERANCE
  #              ep 'var', difference.abs.max
        rescue => err
          TEST_OUT.puts "Error: #{err}"
          TEST_OUT.puts "Error: Variable '#{v}' has failed check in '#@run_name'"
          return false
        end
      end
      return true
    end
  end
end