Class: MesaTestCase

Inherits:
Object
  • Object
show all
Defined in:
lib/mesa_test.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(test: nil, mesa: nil, success_string: '', final_model: 'final.mod', photo: nil, mod: nil) ⇒ MesaTestCase

Returns a new instance of MesaTestCase.



950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
 
# File 'lib/mesa_test.rb', line 950

def initialize(test: nil, mesa: nil, success_string: '',
               final_model: 'final.mod', photo: nil, mod: nil)
  @test_name = test
  @mesa_dir = mesa.mesa_dir
  @mesa = mesa
  @mesa_version = mesa.version_number
  @success_string = success_string
  @final_model = final_model
  @photo = photo
  @failure_type = nil
  @success_type = nil
  @outcome = :not_tested
  @runtime_seconds = 0
  @test_omp_num_threads = 1
  @runtime_minutes = 0
  @total_runtime_seconds = 0
  @retries = 0
  @backups = 0
  @steps = 0
  # 2 (default) means uknown. Updated by running or loading data.
  # 1 means did diffs (not update_checksums; like each_test_run_and_diff)
  # 0 means no diffs (update_checksums; like each_test_run)
  @diff = 2
  # start with nil. Should only be updated to a non-nil value if test is
  # completely successful
  @checksum = nil
  @re_time = nil # rn_time is in the form of @runtime_seconds

  # these only get used with modern versions of both the sdk and the test
  # suite
  @rn_mem = nil
  @re_mem = nil

  # note: this gets overridden for new runs, so this is probably irrelevant
  @summary_text = nil

  # this overrides the submitters choice if it is non-nil
  @compiler = mesa.using_sdk ? 'SDK' : nil
  # only relevant if @compiler is SDK. Gets set during do_one
  @compiler_version = nil

  unless MesaTestCase.modules.include? mod
    raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
                            MesaTestCase.modules.join(', ')
  end
  @mod = mod
  @failure_msg = {
    run_test_string: "#{test_name} run failed: does not match test string",
    final_model: "#{test_name} run failed: final model #{final_model} not " \
      'made.',
    run_checksum: "#{test_name} run failed: checksum for #{final_model} " \
      'does not match after ./rn',
    run_diff: "#{test_name} run failed: diff #{final_model} " \
      'final_check.mod after ./rn',
    photo_file: "#{test_name} restart failed: #{photo} does not exist",
    photo_checksum: "#{test_name} restart failed: checksum for " \
      "#{final_model} does not match after ./re",
    photo_diff: "#{test_name} restart failed: diff #{final_model} " \
      'final_check.mod after ./re',
    compilation: "#{test_name} compilation failed"

  }
  @success_msg = {
    run_test_string: "#{test_name} run: found test string: " \
      "'#{success_string}'",
    run_checksum: "#{test_name} run: checksum for #{final_model} matches " \
      'after ./rn',
    photo_checksum: "#{test_name} restart: checksum for #{final_model} " \
      "matches after ./re #{photo}"
  }

  # validate stuff
  check_mesa_dir
  check_test_case

  @data = {}
  @data_names = []

  # way to output colored text to shell
  @shell = Thor::Shell::Color.new
end

Instance Attribute Details

#backups ⇒ Object (readonly)

Returns the value of attribute backups.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def backups
  @backups
end

#checksum ⇒ Object (readonly)

Returns the value of attribute checksum.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def checksum
  @checksum
end

#compiler ⇒ Object (readonly)

Returns the value of attribute compiler.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def compiler
  @compiler
end

#compiler_version ⇒ Object (readonly)

Returns the value of attribute compiler_version.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def compiler_version
  @compiler_version
end

#data_names ⇒ Object

Returns the value of attribute data_names.



943
944
945
 
# File 'lib/mesa_test.rb', line 943

def data_names
  @data_names
end

#data_types=(value) ⇒ Object

Sets the attribute data_types

Parameters:

  • value —

    the value to set the attribute data_types to.



943
944
945
 
# File 'lib/mesa_test.rb', line 943

def data_types=(value)
  @data_types = value
end

#diff ⇒ Object (readonly)

Returns the value of attribute diff.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def diff
  @diff
end

#failure_msg ⇒ Object (readonly)

Returns the value of attribute failure_msg.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def failure_msg
  @failure_msg
end

#failure_type ⇒ Object

Returns the value of attribute failure_type.



943
944
945
 
# File 'lib/mesa_test.rb', line 943

def failure_type
  @failure_type
end

#final_model ⇒ Object (readonly)

Returns the value of attribute final_model.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def final_model
  @final_model
end

#mesa ⇒ Object (readonly)

Returns the value of attribute mesa.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def mesa
  @mesa
end

#mesa_dir ⇒ Object (readonly)

Returns the value of attribute mesa_dir.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def mesa_dir
  @mesa_dir
end

#mesa_version ⇒ Object (readonly)

Returns the value of attribute mesa_version.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def mesa_version
  @mesa_version
end

#mod ⇒ Object (readonly)

Returns the value of attribute mod.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def mod
  @mod
end

#outcome ⇒ Object

Returns the value of attribute outcome.



943
944
945
 
# File 'lib/mesa_test.rb', line 943

def outcome
  @outcome
end

#photo ⇒ Object (readonly)

Returns the value of attribute photo.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def photo
  @photo
end

#re_mem ⇒ Object (readonly)

Returns the value of attribute re_mem.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def re_mem
  @re_mem
end

#re_time ⇒ Object (readonly)

Returns the value of attribute re_time.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def re_time
  @re_time
end

#retries ⇒ Object (readonly)

Returns the value of attribute retries.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def retries
  @retries
end

#rn_mem ⇒ Object (readonly)

Returns the value of attribute rn_mem.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def rn_mem
  @rn_mem
end

#runtime_minutes ⇒ Object (readonly)

Returns the value of attribute runtime_minutes.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def runtime_minutes
  @runtime_minutes
end

#runtime_seconds ⇒ Object (readonly)

Returns the value of attribute runtime_seconds.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def runtime_seconds
  @runtime_seconds
end

#shell ⇒ Object (readonly)

Returns the value of attribute shell.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def shell
  @shell
end

#steps ⇒ Object (readonly)

Returns the value of attribute steps.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def steps
  @steps
end

#success_msg ⇒ Object (readonly)

Returns the value of attribute success_msg.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def success_msg
  @success_msg
end

#success_string ⇒ Object (readonly)

Returns the value of attribute success_string.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def success_string
  @success_string
end

#success_type ⇒ Object

Returns the value of attribute success_type.



943
944
945
 
# File 'lib/mesa_test.rb', line 943

def success_type
  @success_type
end

#summary_text ⇒ Object (readonly)

Returns the value of attribute summary_text.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def summary_text
  @summary_text
end

#test_name ⇒ Object (readonly)

Returns the value of attribute test_name.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def test_name
  @test_name
end

#test_omp_num_threads ⇒ Object (readonly)

Returns the value of attribute test_omp_num_threads.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def test_omp_num_threads
  @test_omp_num_threads
end

#total_runtime_seconds ⇒ Object (readonly)

Returns the value of attribute total_runtime_seconds.



937
938
939
 
# File 'lib/mesa_test.rb', line 937

def total_runtime_seconds
  @total_runtime_seconds
end

Class Method Details

.modules ⇒ Object 



946
947
948
 
# File 'lib/mesa_test.rb', line 946

def self.modules
  i[star binary]
end

Instance Method Details

#add_datum(datum_name, datum_type) ⇒ Object 



1051
1052
1053
1054
1055
1056
1057
1058
 
# File 'lib/mesa_test.rb', line 1051

def add_datum(datum_name, datum_type)
  unless data_types.include? datum_type.to_sym
    raise InvalidDataType, "Invalid data type: #{datum_type}. Must be one "\
      'of ' + data_types.join(', ') + '.'
  end
  @data[datum_name] = datum_type
  @data_names << datum_name
end

#clean ⇒ Object

based on $MESA_DIR/star/test_suite/each_test_clean, revision 10000



1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
 
# File 'lib/mesa_test.rb', line 1065

def clean
  shell.say("Cleaning #{test_name}", color = :yellow)
  # puts ''
  check_mesa_dir
  check_test_case
  in_dir do
    puts './clean'
    unless bash_execute('./clean')
      raise TestCaseDirError, 'Encountered an error while running ./clean ' \
      "in #{Dir.getwd}."
    end
    shell.say 'Removing all files from LOGS, LOGS1, LOGS2, photos, ' \
      'photos1, and photos2 as well as old test results', color = :blue
    FileUtils.rm_f Dir.glob('LOGS/*')
    FileUtils.rm_f Dir.glob('LOGS1/*')
    FileUtils.rm_f Dir.glob('LOGS2/*')
    FileUtils.rm_f Dir.glob('photos/*')
    FileUtils.rm_f Dir.glob('photos1/*')
    FileUtils.rm_f Dir.glob('photos2/*')

    shell.say 'Removing files binary_history.data, out.txt, ' \
      'test_results.yml, and memory usage files', color = :blue
    FileUtils.rm_f 'binary_history.data'
    FileUtils.rm_f 'out.txt'
    FileUtils.rm_f 'test_results.yml'
    FileUtils.rm_f Dir.glob('mem-*.txt')
    if File.directory? File.join('star_history', 'history_out')
      shell.say 'Removing all files of the form history_out* from ' \
        'star_history', :blue
      FileUtils.rm_f Dir.glob(File.join('star_history', 'history_out', '*'))
    end
    if File.directory? File.join('star_profile', 'profiles_out')
      shell.say 'Removing all files of the form profiles_out* from ' \
        'star_profile', color = :blue
      FileUtils.rm_f Dir.glob(File.join('star_profile', 'profiles_out', '*'))
    end
    shell.say 'Removing .running', color = :blue
    FileUtils.rm_f '.running'
  end
end

#do_one ⇒ Object

based on $MESA_DIR/star/test_suite/each_test_run_and_diff, revision 10000



1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
 
# File 'lib/mesa_test.rb', line 1107

def do_one
  shell.say("Testing #{test_name}", :yellow)
  # puts ''
  test_start = Time.now
  @test_omp_num_threads = omp_num_threads
  if mesa.using_sdk
    version_bin = File.join(ENV['MESASDK_ROOT'], 'bin', 'mesasdk_version')
    # can't use bash_execute because the return value of bash_execute is the
    # exit status of the commmand (true or false), whereas backticks give the
    # output (the version string) as the output
    if File.exist? version_bin
      # newer SDKs have a simple executable
      @compiler_version = `#{version_bin}`.strip
    else
      # older way, call bash on it (file is mesasdk_version.sh)
      @compiler_version = `bash -c #{version_bin + '.sh'}`.strip
    end

    shell.say("Using version #{@compiler_version} of the SDK.", :blue)
  end
  in_dir do
    FileUtils.touch '.running'
    build_and_run
    # report memory usage if it is available
    if File.exist?('mem-rn.txt')
      @rn_mem = File.read('mem-rn.txt').strip.to_i
    end
    if File.exist?('mem-re.txt')
      @re_mem = File.read('mem-re.txt').strip.to_i
    end
    FileUtils.rm '.running'
    puts ''
  end
  test_finish = Time.now
  @total_runtime_seconds = (test_finish - test_start).to_i
end

#load_results ⇒ Object 



1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
 
# File 'lib/mesa_test.rb', line 1178

def load_results
  # loads all parameters from a previous test run, likely for submission
  # purposes
  load_file = File.join(test_case_dir, 'test_results.yml')
  shell.say "Loading data from #{load_file}...", :blue
  unless File.exist? load_file
    shell.say "No such file: #{load_file}. No data loaded.", :red
    return
  end
  data = YAML.safe_load(File.read(load_file), [Symbol])
  @runtime_seconds = data['runtime_seconds'] || @runtime_seconds
  @re_time = data['re_time'] || @re_time
  @total_runtime_seconds = data['total_runtime_seconds'] || @total_runtime_seconds
  @mod = data['module'] || @mod
  @mesa_version = data['mesa_version'] || @mesa_version
  @outcome = data['outcome'] || @outcome
  @test_omp_num_threads = data['omp_num_threads'] || @test_omp_num_threads
  @success_type = data['success_type'] || @success_type
  @failure_type = data['failure_type'] || @failure_type
  @runtime_minutes = data['runtime_minutes'] || @runtime_minutes
  @retries = data['retries'] || @retries
  @backups = data['backups'] || @backups
  @steps = data['steps'] || @steps
  @diff = data['diff'] || @diff
  @checksum = data['checksum'] || @checksum
  @rn_mem = data['rn_mem'] || @rn_mem
  @re_mem = data['re_mem'] || @re_mem
  @summary_text = data['summary_text'] || @summary_text
  @compiler = data['compiler'] || @compiler

  @compiler_version = data['compiler_version'] || @compiler_version

  # convert select data to symbols since that is how they are used
  @outcome = @outcome.to_sym if @outcome
  @success_type = @success_type.to_sym if @success_type
  @failure_type = @failure_type.to_sym if @failure_type

  shell.say "Done loading data from #{load_file}.\n", :green
end

#load_summary_data ⇒ Object 



1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
 
# File 'lib/mesa_test.rb', line 1218

def load_summary_data
  begin
    out_data = parse_out
    @summary_text = get_summary_text
  rescue Errno::ENOENT
    shell.say "\nError loading data from #{out_file}. No summary data "\
              'loaded. Proceeding anyway.', :red
  else
    @runtime_minutes = out_data[:runtime_minutes]
    @retries = out_data[:retries]
    @backups = out_data[:backups]
    @steps = out_data[:steps]
  end
end

#log_results ⇒ Object 



1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
 
# File 'lib/mesa_test.rb', line 1144

def log_results
  # gets all parameters that would be submitted as well as computer
  # information and dumps to a yml file in the test case directory
  save_file = File.join(test_case_dir, 'test_results.yml')
  shell.say "Logging test results to #{save_file}...", :blue
  res = {
    'test_case' => test_name,
    'module' => mod,
    'runtime_seconds' => runtime_seconds,
    're_time' => re_time,
    'total_runtime_seconds' => total_runtime_seconds,
    'mesa_version' => mesa_version,
    'outcome' => outcome,
    'omp_num_threads' => test_omp_num_threads,
    'success_type' => success_type,
    'failure_type' => failure_type,
    'runtime_minutes' => runtime_minutes,
    'retries' => retries,
    'backups' => backups,
    'steps' => steps,
    'diff' => diff,
    'checksum' => checksum,
    'rn_mem' => rn_mem,
    're_mem' => re_mem,
    'summary_text' => summary_text
  }
  if compiler == 'SDK'
    res['compiler'] = 'SDK'
    res['compiler_version'] = compiler_version
  end
  File.open(save_file, 'w') { |f| f.write(YAML.dump(res)) }
  shell.say "Successfully saved results to file #{save_file}.\n", :green
end

#omp_num_threads ⇒ Object 



1060
1061
1062
 
# File 'lib/mesa_test.rb', line 1060

def omp_num_threads
  ENV['OMP_NUM_THREADS'].to_i || 1
end

#parse_out ⇒ Object 



1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
 
# File 'lib/mesa_test.rb', line 1233

def parse_out
  runtime_minutes = 0
  retries = 0
  backups = 0
  steps = 0
  run_summaries.each do |summary|
    summary =~ /^\s*runtime\s*\(minutes\),\s+retries,\s+backups,\ssteps\s+(\d+\.?\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
    runtime_minutes += $1.to_f
    retries += $2.to_i
    backups += $3.to_i
    steps += $4.to_i
  end
  {runtime_minutes: runtime_minutes, retries: retries, backups: backups,
   steps: steps}
end

#passed? ⇒ Boolean

Returns:

  • (Boolean) 


1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
 
# File 'lib/mesa_test.rb', line 1032

def passed?
  if @outcome == :pass
    true
  elsif @outcome == :fail
    false
  else
    raise TestCaseDirError, 'Cannot determine pass/fail status of ' \
    "#{test_name} yet."
  end
end

#test_case_dir ⇒ Object 



1047
1048
1049
 
# File 'lib/mesa_test.rb', line 1047

def test_case_dir
  File.join(test_suite_dir, test_name)
end

#test_suite_dir ⇒ Object 



1043
1044
1045
 
# File 'lib/mesa_test.rb', line 1043

def test_suite_dir
  mesa.test_suite_dir(mod: @mod)
end