Class: Modsvaskr::InGameTestsRunner

Inherits:
Object
  • Object
show all
Includes:
Logger
Defined in:
lib/modsvaskr/in_game_tests_runner.rb

Overview

Class getting a simple API to handle tests that are run in-game

Instance Method Summary collapse

Methods included from Logger

#log, #out, #wait_for_user_enter

Constructor Details

#initialize(config, game) ⇒ InGameTestsRunner

Constructor. Default values are for a standard Skyrim SE installation.

Parameters

  • config (Config): Main configuration

  • game (Game): Game for which we run tests



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/modsvaskr/in_game_tests_runner.rb', line 22

def initialize(config, game)
  @config = config
  @game = game
  auto_test_esp = "#{@game.path}/Data/AutoTest.esp"
  # Ordered list of available in-game test suites
  # Array<Symbol>
  @available_tests_suites =
    if File.exist?(auto_test_esp)
      Base64.decode64(
        ElderScrollsPlugin.new(auto_test_esp).
          to_json[:sub_chunks].
          find { |chunk| chunk[:decoded_header][:label] == 'QUST' }[:sub_chunks].
          find do |chunk|
            chunk[:sub_chunks].any? { |sub_chunk| sub_chunk[:name] == 'EDID' && sub_chunk[:data] =~ /AutoTest_ScriptsQuest/ }
          end[:sub_chunks].
          find { |chunk| chunk[:name] == 'VMAD' }[:data]
      ).scan(/AutoTest_Suite_(\w+)/).flatten.map { |tests_suite| tests_suite.downcase.to_sym }
    else
      log "[ In-game testing #{@game.name} ] - Missing file #{auto_test_esp}. In-game tests will be disabled. Please install the AutoTest mod."
      []
    end
  log "[ In-game testing #{@game.name} ] - #{@available_tests_suites.size} available in-game tests suites: #{@available_tests_suites.join(', ')}"
end

Instance Method Details

#run(selected_tests) ⇒ Object

Run in-game tests in a loop until they are all tested

Parameters

  • selected_tests (Hash<Symbol, Array<String> >): Ordered list of in-game tests to be run, per in-game tests suite

  • Proc: Code called when a in-game test status has changed

    • Parameters

      • in_game_tests_suite (Symbol): The in-game tests suite for which test statuses have changed

      • in_game_tests_statuses (Hash<String,String>): Tests statuses, per test name



54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
 
# File 'lib/modsvaskr/in_game_tests_runner.rb', line 54

def run(selected_tests)
  unknown_tests_suites = selected_tests.keys - @available_tests_suites
  log "[ In-game testing #{@game.name} ] - !!! The following in-game tests suites are not supported: #{unknown_tests_suites.join(', ')}" unless unknown_tests_suites.empty?
  tests_to_run = selected_tests.except(*unknown_tests_suites)
  return if tests_to_run.empty?

  FileUtils.mkdir_p "#{@game.path}/Data/SKSE/Plugins/StorageUtilData"
  tests_to_run.each do |tests_suite, tests|
    # Write the JSON file that contains the list of tests to run
    File.write(
      "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{tests_suite}_Run.json",
      JSON.pretty_generate(
        'stringList' => {
          'tests_to_run' => tests
        }
      )
    )
    # Clear the AutoTest test statuses that we are going to run
    statuses_file = "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{tests_suite}_Statuses.json"
    next unless File.exist?(statuses_file)

    File.write(
      statuses_file,
      JSON.pretty_generate('string' => JSON.parse(File.read(statuses_file))['string'].delete_if { |test_name, _test_status| tests.include?(test_name) })
    )
  end
  auto_test_config_file = "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_Config.json"
  # Write the JSON file that contains the configuration of the AutoTest tests runner
  File.write(
    auto_test_config_file,
    JSON.pretty_generate(
      'string' => {
        'on_start' => 'run',
        'on_stop' => 'exit'
      }
    )
  )
  out ''
  out '=========================================='
  out '= In-game tests are about to be launched ='
  out '=========================================='
  out ''
  out 'Here is what you need to do once the game will be launched (don\'t launch it by yourself, the test framework will launch it for you):'
  out '* Load the game save you want to test (or start a new game).'
  out ''
  out 'This will execute all in-game tests automatically.'
  out ''
  out 'It is possible that the game crashes during tests:'
  out '* That\'s a normal situation, as tests don\'t mimick a realistic gaming experience, and the Bethesda engine is not meant to be stressed like that.'
  out '* In case of game crash (CTD), the Modsvaskr test framework will relaunch it automatically and resume testing from when it crashed.'
  out '* In case of repeated CTD on the same test, the Modsvaskr test framework will detect it and skip the crashing test automatically.'
  out '* In case of a game freeze without CTD, the Modsvaskr test framework will detect it after a few minutes and automatically kill the game before re-launching it to resume testing.'
  out ''
  out 'If you want to interrupt in-game testing: invoke the console with ~ key and type stop_tests followed by Enter.'
  out ''
  out 'Press enter to start in-game testing (this will lauch your game automatically)...'
  wait_for_user_enter
  last_time_tests_changed = nil
  with_auto_test_monitoring(
    on_auto_test_statuses_diffs: proc do |in_game_tests_suite, in_game_tests_statuses|
      yield in_game_tests_suite, in_game_tests_statuses
      last_time_tests_changed = Time.now
    end
  ) do
    # Loop on (re-)launching the game when we still have tests to perform
    idx_launch = 0
    loop do
      # Check which test is supposed to run first, as it will help in knowing if it fails or not.
      first_tests_suite_to_run = nil
      first_test_to_run = nil
      current_tests_statuses = check_auto_test_statuses
      @available_tests_suites.each do |tests_suite|
        next unless tests_to_run.key?(tests_suite)

        found_test_ok =
          if current_tests_statuses.key?(tests_suite)
            # Find the first test that would be run (meaning the first one having no status, or status 'started')
            tests_to_run[tests_suite].find do |test_name|
              found_test_name, found_test_status = current_tests_statuses[tests_suite].find { |(current_test_name, _current_test_status)| current_test_name == test_name }
              found_test_name.nil? || found_test_status == 'started'
            end
          else
            # For sure the first test of this suite will be the first one to run
            tests_to_run[tests_suite].first
          end
        next unless found_test_ok

        first_tests_suite_to_run = tests_suite
        first_test_to_run = found_test_ok
        break
      end
      if first_tests_suite_to_run.nil?
        log "[ In-game testing #{@game.name} ] - No more test to be run."
        break
      else
        log "[ In-game testing #{@game.name} ] - First test to run should be #{first_tests_suite_to_run} / #{first_test_to_run}."
        # Launch the game to execute AutoTest
        @game.launch(autoload: idx_launch.zero? ? false : 'auto_test')
        idx_launch += 1
        log "[ In-game testing #{@game.name} ] - Start monitoring in-game testing..."
        last_time_tests_changed = Time.now
        while @game.running?
          check_auto_test_statuses
          # If the tests haven't changed for too long, consider the game has frozen, but not crashed. So kill it.
          if Time.now - last_time_tests_changed > @game.timeout_frozen_tests_secs
            log "[ In-game testing #{@game.name} ] - Last time in-game tests statuses have changed is #{last_time_tests_changed.strftime('%F %T')}. Consider the game is frozen, so kill it."
            @game.kill
          else
            sleep @game.tests_poll_secs
          end
        end
        last_test_statuses = check_auto_test_statuses
        # Log latest statuses
        log "[ In-game testing #{@game.name} ] - End monitoring in-game testing. In-game test statuses after game run:"
        last_test_statuses.each do |tests_suite, statuses_for_type|
          log "[ In-game testing #{@game.name} ] - [ #{tests_suite} ] - #{statuses_for_type.select { |(_name, status)| status == 'ok' }.size} / #{statuses_for_type.size}"
        end
        # Check for which reason the game has stopped, and eventually end the testing session.
        # Careful as this JSON file can be written by Papyrus that treat strings as case insensitive.
        # cf. https://github.com/xanderdunn/skaar/wiki/Common-Tasks
        auto_test_config = JSON.parse(File.read(auto_test_config_file))['string'].to_h { |key, value| [key.downcase, value.downcase] }
        if auto_test_config['stopped_by'] == 'user'
          log "[ In-game testing #{@game.name} ] - Tests have been stopped by user."
          break
        end
        if auto_test_config['tests_execution'] == 'end'
          log "[ In-game testing #{@game.name} ] - Tests have finished running."
          break
        end
        # From here we know that the game has either crashed or has been killed.
        # This is an abnormal termination of the game.
        # We have to know if this is due to a specific test that fails deterministically, or if it is the engine being unstable.
        # Check the status of the first test that should have been run to know about it.
        first_test_status = nil
        _found_test_name, first_test_status = last_test_statuses[first_tests_suite_to_run].find { |(current_test_name, _current_test_status)| current_test_name == first_test_to_run } if last_test_statuses.key?(first_tests_suite_to_run)
        if first_test_status == 'ok'
          # It's not necessarily deterministic.
          # We just have to go on executing next tests.
          log "[ In-game testing #{@game.name} ] - Tests session has finished in error, certainly due to the game's normal instability. Will resume testing."
        else
          # The first test doesn't pass.
          # We need to mark it as failed, then remove it from the runs.
          log "[ In-game testing #{@game.name} ] - First test #{first_tests_suite_to_run} / #{first_test_to_run} is in error status: #{first_test_status}. Consider it failed and skip it for next run."
          # If the test was started but failed before setting its status to something else then change the test status in the JSON file directly so that AutoTest does not try to re-run it.
          if first_test_status == 'started' || first_test_status == '' || first_test_status.nil?
            File.write(
              "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{first_tests_suite_to_run}_Statuses.json",
              JSON.pretty_generate(
                'string' => ((last_test_statuses[first_tests_suite_to_run] || []) + [[first_test_to_run, '']]).to_h do |(test_name, test_status)|
                  [
                    test_name,
                    test_name == first_test_to_run ? 'failed_ctd' : test_status
                  ]
                end
              )
            )
            # Notify the callbacks updating test statuses
            check_auto_test_statuses
          end
        end
        # We will start again. Leave some time to interrupt if we want.
        if @config.no_prompt
          out 'Start again automatically as no_prompt has been set.'
        else
          # First, flush stdin of any pending character
          $stdin.getc until select([$stdin], nil, nil, 2).nil?
          out "We are going to start again in #{@game.timeout_interrupt_tests_secs} seconds. Press Enter now to interrupt it."
          key_pressed =
            begin
              Timeout.timeout(@game.timeout_interrupt_tests_secs) { $stdin.gets }
            rescue Timeout::Error
              nil
            end
          if key_pressed
            log "[ In-game testing #{@game.name} ] - Run interrupted by user."
            # TODO: Remove AutoTest start on load: it has been interrupted by the user, so we should not keep it in case the user launches the game by itself.
            break
          end
        end
      end
    end
  end
end