Class: Flaky::Run

Inherits:
Object
  • Object
show all
Includes:
Color
Defined in:
lib/flaky/run.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Color

#cyan, #green, #red

Constructor Details

#initialize(result_dir_postfix = '') ⇒ Run

Returns a new instance of Run.



41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/flaky/run.rb', line 41

def initialize result_dir_postfix=''
  @tests = {}
  @start_time = Time.now

  result_dir = '/tmp/flaky/' + result_dir_postfix
  # rm -rf result_dir
  FileUtils.rm_rf result_dir
  FileUtils.mkdir_p result_dir

  @result_dir = result_dir
  @result_file = File.join result_dir, 'result.txt'
  @fail_file = File.join result_dir, 'fail.txt'
end

Instance Attribute Details

#result_dir ⇒ Object (readonly)

Returns the value of attribute result_dir.



39
40
41
 
# File 'lib/flaky/run.rb', line 39

def result_dir
  @result_dir
end

#result_file ⇒ Object (readonly)

Returns the value of attribute result_file.



39
40
41
 
# File 'lib/flaky/run.rb', line 39

def result_file
  @result_file
end

#tests ⇒ Object (readonly)

Returns the value of attribute tests.



39
40
41
 
# File 'lib/flaky/run.rb', line 39

def tests
  @tests
end

Instance Method Details

#_execute(run_cmd, test_name, runs, appium, sauce) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/flaky/run.rb', line 124

def _execute run_cmd, test_name, runs, appium, sauce
  # must capture exit code or log is an array.
  result = /\d+ runs, \d+ assertions, \d+ failures, \d+ errors, \d+ skips/
  success = /0 failures, 0 errors, 0 skips/
  passed = true

  exit_code = -1
  timedout = false
  rake_pid = -1
  # we need the minitest log in memory to scan for results.
  log = ''
  tmp_ruby_log = '/tmp/flaky/ruby_log_tmp.txt'
  File.delete(tmp_ruby_log) if File.exists? tmp_ruby_log

  flaky_logs_txt = '/tmp/flaky_logs.txt'
  File.delete flaky_logs_txt if File.exist? flaky_logs_txt
  tail_cmd = "tail -f -n1 /Users/#{ENV['USER']}/Library/Logs/iOS\\ Simulator/7.0.3/system.log > #{flaky_logs_txt}"
  tail_cmd = "adb logcat > #{flaky_logs_txt}" if !sauce && !appium.ios

  tail_system_log = Flaky::Cmd.new tail_cmd
  begin
    ten_minutes = 10 * 60
    timeout ten_minutes do
      rake = Flaky::Cmd.new run_cmd
      rake_pid = rake.pid

      while process_exists rake_pid
        begin
          # readpartial throws end of file reached error
          new_out = rake.out.readpartial 999_999 # blocks on 0 data
          log += new_out

          File.open(tmp_ruby_log, 'a') do |f|
            f.write new_out
          end
        rescue
        end
      end

      # must write rake.err. it's not included in rake.out
      begin
        new_err = rake.err.read_nonblock 999_999

        log += new_err if new_err
        File.open(tmp_ruby_log, 'a') do |f|
          f.write new_err
        end if new_err
      rescue
      end
    end
  rescue Exception => e
    timedout = true
    passed = false
    # after_run in run.rb is triggered by sigint
    Process.kill :SIGINT, rake_pid

    begin
      two_minutes = 2 * 60
      timeout two_minutes do
        Process::waitpid rake_pid
      end
    rescue # if the process still isn't done after sigint, use sigkill
      Process.kill :SIGKILL, rake_pid
    end
  end

  # waitpid may throw if the pid doesn't exist by the time we're ready to wait.
  begin
    tail_system_log_pid = tail_system_log.pid
    Process.kill :SIGKILL, tail_system_log_pid
    Process::waitpid tail_system_log_pid
  rescue
  end

  unless timedout
    found_results = log.scan result
    # all result instances must match success
    found_results.each do |result|
      # runs must be >= 1. 0 runs mean no tests were run.
      r_count = result.match /(\d+) runs/
      runs_not_zero = r_count && r_count[1] && r_count[1].to_i > 0 ? true : false

      unless result.match(success) && runs_not_zero
        passed = false
        break
      end
    end

    # no results found.
    passed = false if found_results.length <= 0
  end
  pass_str = passed ? 'pass' : 'fail'
  test = @tests[test_name]
  # save log
  if passed
    pass = test[:pass] += 1
    postfix = "pass_#{pass}"
  else
    fail = test[:fail] += 1
    postfix = "fail_#{fail}"
    test[:timedout] = true if timedout
  end

  postfix = "#{runs}_#{test_name}_" + postfix
  postfix = '0' + postfix if runs <= 9

  log_file = LogArtifact.new result_dir: result_dir, pass_str: pass_str, test_name: test_name

  # File.open 'w' will not create folders. Use mkdir_p before.
  test_file_path = log_file.name("#{postfix}.txt")
  FileUtils.mkdir_p File.dirname(test_file_path)
  # html Ruby test log
  File.open(test_file_path, 'w') do |f|
    f.write log
  end

  unless sauce
    movie_path = log_file.name("#{postfix}.mov")
    FileUtils.mkdir_p File.dirname(movie_path)
    movie_src = '/tmp/video.mov'
    if File.exists?(movie_src)
      unless Flaky.no_video
        # save movie on failure
        FileUtils.copy movie_src, movie_path if !passed
      end
      # always clean up movie
      File.delete movie_src if File.exists? movie_src
    end

    # save .TIMED_OUT.txt in timeout fails
    if timedout
      timeout_path = log_file.name("#{postfix}.TIMED_OUT.txt")
      FileUtils.mkdir_p File.dirname(timeout_path)
      File.open(timeout_path, 'w') {}
    end

    src_system_log = '/tmp/flaky_logs.txt'
    if File.exists? src_system_log
      # postfix is required! or the log will be saved to an incorrect path
      system_log_path = log_file.name("#{postfix}.system.txt")
      FileUtils.mkdir_p File.dirname(system_log_path)
      FileUtils.copy_file src_system_log, system_log_path
      File.delete src_system_log if File.exists? src_system_log
    end

    # appium server log
    appium_server_path = log_file.name("#{postfix}.appium.txt")
    FileUtils.mkdir_p File.dirname(appium_server_path)

    tmp_file = appium.flush_buffer
    if File.exists?(tmp_file) && !tmp_file.nil? && !tmp_file.empty?
      FileUtils.copy_file tmp_file, appium_server_path
    end
    File.delete tmp_file if File.exists? tmp_file
    # also delete the temp ruby log
    File.delete tmp_ruby_log if File.exists? tmp_ruby_log

    # copy app logs
    app_logs = '/tmp/flaky_tmp_log_folder'
    dest_dir = File.dirname(appium_server_path)
    if File.exists? app_logs
      Dir.glob(File.join(app_logs, '*')).each { |f| FileUtils.cp f, dest_dir }
    end
    FileUtils.rm_rf app_logs

    # copy android coverage
    coverage_folder = '/tmp/flaky/coverage'
    FileUtils.mkdir_p coverage_folder
    coverage_file = '/tmp/coverage.ec'
    if File.exists? '/tmp/coverage.ec'
      FileUtils.cp coverage_file, File.join(coverage_folder, "#{Time.now.to_i}.ec")
      File.delete coverage_file
    end
  end

  passed
end

#collect_crashes(array) ⇒ Object 



302
303
304
305
306
307
 
# File 'lib/flaky/run.rb', line 302

def collect_crashes array
  Dir.glob(File.join(Dir.home, '/Library/Logs/DiagnosticReports/*.crash')) do |crash|
    array << crash
  end
  array
end

#execute(opts = {}) ⇒ Object 



309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
 
# File 'lib/flaky/run.rb', line 309

def execute opts={}
  run_cmd = opts[:run_cmd]
  test_name = opts[:test_name]
  appium = opts[:appium]
  sauce = opts[:sauce]

  old_crash_files = []
  # appium is nil when on sauce
  if !sauce && appium
    collect_crashes old_crash_files
  end

  raise 'must pass :run_cmd' unless run_cmd
  raise 'must pass :test_name' unless test_name
  # local appium is not required when running on Sauce
  raise 'must pass :appium' unless appium || sauce

  test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0, timedout: false}
  runs = test[:runs] += 1

  passed = _execute run_cmd, test_name, runs, appium, sauce
  unless sauce
    print cyan("\n #{test_name} ") if @last_test.nil? ||
        @last_test != test_name

    print passed ? green(' ✓') : red(' ✖')
  else
    print cyan("\n #{test_name} ")
    print passed ? green(' ✓') : red(' ✖')
    print " https://saucelabs.com/tests/#{File.read('/tmp/appium_lib_session').chomp}\n"
  end

  # androids adb may crash also and it ends up in the same location as iOS.
  # appium is nil when running on Sauce
  if !sauce && appium
    new_crash_files = []
    collect_crashes new_crash_files

    new_crash_files = new_crash_files - old_crash_files
    if new_crash_files.length > 0
      File.open('/tmp/flaky/crashes.txt', 'a') do |f|
        f.puts '--'
        f.puts "Test: #{test_name} crashed on #{appium.ios ? 'ios' : 'android'}:"
        new_crash_files.each { |crash| f.puts crash }
        f.puts '--'
      end
    end
  end

  @last_test = test_name
  passed
end

#process_exists(pid) ⇒ Object 



115
116
117
118
119
120
121
122
 
# File 'lib/flaky/run.rb', line 115

def process_exists pid
  begin
    Process.waitpid(pid, Process::WNOHANG)
    true
  rescue
    false
  end
end

#report(opts = {}) ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/flaky/run.rb', line 55

def report opts={}
  save_file = opts.fetch :save_file, true
  puts "\n" * 2
  success = ''
  failure = ''
  failure_name_only = ''
  total_success = 0
  total_failure = 0
  @tests.each do |name, stats|
    runs = stats[:runs]
    pass = stats[:pass]
    fail = stats[:fail]
    timedout = ''
    timedout = '-- TIMED OUT' if stats[:timedout] == true
    line = "#{name}, runs: #{runs}, pass: #{pass}," +
        " fail: #{fail} #{timedout}\n"
    if fail > 0 && pass <= 0
      failure_name_only += "#{File.basename(name)}\n"
      failure += line
      total_failure += 1
    else
      success += line
      total_success += 1
    end
  end

  total_tests = total_success + total_failure
  out = "#{total_tests} Tests\n\n"
  out += "Failure (#{total_failure}):\n#{failure}\n" unless failure.empty?
  out += "Success (#{total_success}):\n#{success}" unless success.empty?

  time_now = Time.now
  duration = time_now - @start_time
  duration = ChronicDuration.output(duration.round) || '0s'
  out += "\nFinished in #{duration}\n"
  time_format = '%b %d %l:%M %P'
  time_format2 = '%l:%M %P'
  out += "#{@start_time.strftime(time_format)} - #{time_now.strftime(time_format2)}"

  month_day_year = Time.now.strftime '%-m/%-d/%Y'
  success_percent = (total_success.to_f/total_tests.to_f*100).round(2)
  success_percent = 100 if total_failure <= 0
  google_docs_line = [month_day_year, total_tests, total_failure, total_success, success_percent].join("\t")
  out += "\n#{google_docs_line}"
  out += "\n--\n"

  if save_file
    File.open(@fail_file, 'w') do |f|
      f.puts failure_name_only
    end

    # overwrite file
    File.open(@result_file, 'w') do |f|
      f.puts out
    end
  end

  puts out
end