Module: Flaky

Defined in:
lib/flaky.rb,
lib/flaky/run.rb,
lib/flaky/appium.rb,
lib/screen_recording.rb,
lib/flaky/applescript.rb,
lib/flaky/run/one_test.rb,
lib/flaky/run/two_pass.rb,
lib/flaky/run/all_tests.rb,
lib/flaky/run/from_file.rb

Defined Under Namespace

Modules: Color Classes: Appium, AppleScript, Cmd, LogArtifact, Run

Constant Summary collapse

VERSION = 
'0.1.2'
DATE = 
'2015-04-28'

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.no_video ⇒ Object

Returns the value of attribute no_video.



16
17
18
 
# File 'lib/flaky.rb', line 16

def no_video
  @no_video
end

Class Method Details

.capture_ios_app_log(app_name) ⇒ Object

app_name for example MyApp.app



12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/screen_recording.rb', line 12

def capture_ios_app_log app_name
  begin
    app_glob = "/Users/#{ENV['USER']}/Library/Application Support/iPhone Simulator/7.0.3/Applications/*/#{app_name}"
    app_folder = File.dirname Dir.glob(app_glob).first

    tmp_log_folder = '/tmp/flaky_tmp_log_folder'
    FileUtils.rm_rf tmp_log_folder if File.exists? tmp_log_folder
    FileUtils.mkdir_p tmp_log_folder

    log_glob = File.join app_folder, 'Library/Caches/Logs/*'
    Dir.glob(log_glob).each { |log| FileUtils.cp log, tmp_log_folder }
  rescue # folder may not exist. or there could be no longs
  end
end

.run_all_tests(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/flaky/run/all_tests.rb', line 3

def self.run_all_tests opts={}
  raise 'Must pass :count and :os' unless opts && opts[:count] && opts[:os]

  count = opts[:count].to_i
  os = opts[:os]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd
  rakefile = File.expand_path(File.join(current_dir, 'Rakefile'))
  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists? rakefile
  flaky_txt = File.expand_path(File.join(current_dir, 'flaky.txt'))
  parsed = TOML.load File.read flaky_txt
  puts "flaky.txt: #{parsed}"
  android_dir = parsed['android']
  ios_dir = parsed['ios']
  glob = parsed.fetch 'glob', '**/*.rb'

  active_dir = is_android ? android_dir : ios_dir
  final_path = File.expand_path File.join current_dir, active_dir, glob
  puts "Globbing: #{final_path}"

  Dir.glob(final_path) do |test_file|
    raise "#{test_file} does not exist." unless File.exist?(test_file)
    test_file = File.expand_path test_file

    test_name = test_file.sub(File.expand_path(File.join(current_dir, active_dir)), '')
    # remove leading /
    test_name.sub!(test_name.match(/^\//).to_s, '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{test_file}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.run_from_file(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/flaky/run/from_file.rb', line 3

def self.run_from_file opts={}
  raise 'Must pass :count, :os, and :file' unless opts && opts[:count] && opts[:os] && opts[:file]

  count = opts[:count].to_i
  os = opts[:os]
  file = opts[:file]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)
  raise ':file must be a string' unless file.kind_of?(String)

  raise "#{file} doesn't exist" unless File.exists? file
  tests = File.readlines(file).map { |line| File.basename(line.chomp, '.*') }
  resolved_paths = []
  # Convert file names into full paths
  current_dir = Dir.pwd
  Dir.glob(File.join current_dir, 'appium', os, 'specs', '**/*.rb') do |test_file|
    if tests.include? File.basename(test_file, '.*')
      resolved_paths << File.expand_path(test_file)
    end
  end

  if tests.length != resolved_paths.length
    missing_tests = []
    tests.each do |test|
      missing_tests << test unless File.exists? test
    end
    raise "Missing tests #{missing_tests}"
  end

  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  resolved_paths.each do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count.times do
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.run_one_test(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/flaky/run/one_test.rb', line 3

def self.run_one_test opts={}
  raise 'Must pass :count and :name' unless opts && opts[:count] && opts[:os] && opts[:name]

  count = opts[:count].to_i
  os = opts[:os]
  name = opts[:name]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)
  raise ':name must be a string' unless name.kind_of?(String)

  # ensure file name does not contain an extension
  # don't expand the path because it's joined and expanded in final_path.
  name = File.join(File.dirname(name), File.basename(name, '.*'))

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  flaky = Flaky::Run.new
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd

  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))
  flaky_txt = File.expand_path(File.join(current_dir, 'flaky.txt'))
  parsed = TOML.load File.read flaky_txt
  puts "flaky.txt: #{parsed}"
  android_dir = parsed['android']
  ios_dir = parsed['ios']
  active_dir = is_android ? android_dir : ios_dir
  final_path = File.expand_path File.join current_dir, active_dir, name + '.rb'
  test_file = ''
  Dir.glob(final_path) do |file|
    test_file = file
  end

  raise "#{test_file} does not exist." unless File.exists?(test_file)

  test_name = test_file.sub(File.expand_path(File.join(current_dir, active_dir)), '')
  # remove leading /
  test_name.sub!(test_name.match(/^\//).to_s, '')
  test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

  count.times do
    appium.start unless running_on_sauce
    run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{test_file}',#{Flaky.no_video}]"
    flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
  end

  appium.stop unless running_on_sauce
  flaky.report
end

.screen_recording_binary ⇒ Object 



27
28
29
 
# File 'lib/screen_recording.rb', line 27

def screen_recording_binary
  @screen_recording_binary ||= File.expand_path('../screen-recording', __FILE__)
end

.screen_recording_start(opts = {}) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
# File 'lib/screen_recording.rb', line 31

def screen_recording_start opts={}
  return if Flaky.no_video

  os = opts[:os]
  path = opts[:path]
  raise ':os is required' unless os
  raise ':path is required' unless path

  raise 'Invalid os. Must be ios or android' unless %w[ios android].include? os
  raise 'Invalid path. Must end with .mov' unless File.extname(path) == '.mov'
  raise 'Invalid path. Must not be a dir' if File.exists?(path) && File.directory?(path)

  # ensure we have exactly one screen-recording process
  # wait for killall to complete
  Process::waitpid(spawn('killall', '-9', 'screen-recording', :in => '/dev/null', :out => '/dev/null', :err => '/dev/null'))

  File.delete(path) if File.exists? path

  pid = spawn(screen_recording_binary, os, path,
              :in => '/dev/null', :out => '/dev/null', :err => '/dev/null')
  pid
end

.screen_recording_stop(pid) ⇒ Object 



54
55
56
57
58
59
60
 
# File 'lib/screen_recording.rb', line 54

def screen_recording_stop pid
  Process.kill(:SIGINT, pid)
  # Must wait 5 seconds for the video to end.
  # If we don't wait, the movie will be corrupt.
  # See: https://github.com/bootstraponline/screen_recording/blob/master/screen-recording/main.m#L137
  sleep 5
end

.two_pass(opts = {}) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/flaky/run/two_pass.rb', line 3

def self.two_pass opts={}
  raise 'Must pass :count and :os' unless opts && opts[:count] && opts[:os]

  count = opts[:count].to_i
  os = opts[:os]

  raise ':count must be an int' unless count.kind_of?(Integer)
  raise ':os must be a string' unless os.kind_of?(String)

  count1 = 1
  count2 = count

  running_on_sauce = ENV['SAUCE_USERNAME'] ? true : false
  FileUtils.rm_rf '/tmp/flaky'
  result_dir_postfix = '1' # /tmp/flaky/1
  flaky = Flaky::Run.new(result_dir_postfix)
  is_android = os.strip.downcase == 'android'
  appium = Appium.new(android: is_android) unless running_on_sauce

  current_dir = Dir.pwd
  raise "Rakefile doesn't exist in #{current_dir}" unless File.exists?(File.join(current_dir, 'Rakefile'))

  # run all tests once
  Dir.glob(File.join current_dir, 'appium', os, 'specs', '**/*.rb') do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count1.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute(run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce)
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report

  # ---

  # now run only the failures count2 times
  fails = File.read(File.join('/tmp/flaky/', result_dir_postfix, 'fail.txt'))

  result_dir_postfix = '2' # /tmp/flaky/1
  flaky = Flaky::Run.new(result_dir_postfix)
  appium = Appium.new unless running_on_sauce

  fails.split("\n").each do |test_file|
    file = test_file
    name = File.basename file, '.*'

    raise "#{test_file} does not exist." if file.empty?

    test_name = file.sub(current_dir + '/appium/', '')
    test_name = File.join(File.dirname(test_name), File.basename(test_name, '.*'))

    count2.times do
      File.open('/tmp/flaky/current.txt', 'a') { |f| f.puts "Running: #{test_name} on #{os}" }
      appium.start unless running_on_sauce
      run_cmd = "cd #{current_dir}; rake #{os.downcase}['#{name}',#{Flaky.no_video}]"
      passed = flaky.execute run_cmd: run_cmd, test_name: test_name, appium: appium, sauce: running_on_sauce
      break if passed # move onto the next test after one successful run
    end
  end

  appium.stop unless running_on_sauce
  flaky.report
end