Class: Telly::TestRailTeller

Inherits:
Object
  • Object
show all
Defined in:
lib/telly/test_rail_teller.rb

Defined Under Namespace

Classes: MissingTestRailId

Constant Summary collapse

TESTRAIL_URL = 
'https://testrail.ops.puppetlabs.net/'
CREDENTIALS_FILE = 
'~/.testrail_credentials.yaml'
TESTCASE_ID_REGEX =

Used for extracted the test case ID from beaker scripts

/.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
PASSED =

Testrail Status IDs

1
BLOCKED = 
2
FAILED = 
5

Instance Method Summary collapse

Instance Method Details

#beaker_test_path(junit_file_path, junit_result) ⇒ String

Calculates the path to a beaker test file by combining the junit file path with the test name from the junit results. Makes the assumption that junit folder that beaker creates will always be 2 directories up from the beaker script base directory. TODO somewhat hacky, maybe a config/command line option

Examples:

load_junit_results(“~/junit/latest/beaker_junit.xml”)

Parameters:

  • junit_file_path (String) —

    Path to a junit xml file

  • junit_result (String) —

    Path to a junit xml file

Returns:

  • (String) —

    The path to the beaker script from the junit test result



296
297
298
299
300
301
 
# File 'lib/telly/test_rail_teller.rb', line 296

def beaker_test_path(junit_file_path, junit_result)
  beaker_folder_path = junit_result[:classname]
  test_filename = junit_result[:name]

  File.join(File.dirname(junit_file_path), "../../../", beaker_folder_path, test_filename)
end

#do_stub_test(credentials) ⇒ Object 



42
43
44
45
46
 
# File 'lib/telly/test_rail_teller.rb', line 42

def do_stub_test(credentials)
  api = get_testrail_api(credentials)

  api.send_post
end

#get_testrail_api(credentials) ⇒ TestRail::APIClient

Returns a testrail API object that talks to testrail

Examples:

api = get_testrail_api(load_credentials)

Parameters:

  • credentials (Hash) —

    A hash containing at least two keys, testrail_username and testrail_password

Returns:



110
111
112
113
114
115
116
 
# File 'lib/telly/test_rail_teller.rb', line 110

def get_testrail_api(credentials)
  client = TestRail::APIClient.new(TESTRAIL_URL)
  client.user = credentials["testrail_username"]
  client.password = credentials["testrail_password"]

  return client
end

#load_credentials(credentials_file) ⇒ Hash

Load testrail credentials from file

Examples:

password = load_credentials()

Returns:

  • (Hash) —

    Contains testrail_username and testrail_password



89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/telly/test_rail_teller.rb', line 89

def load_credentials(credentials_file)
  begin
    YAML.load_file(File.expand_path(credentials_file))
  rescue
    puts "Error: Could not find #{credentials_file}"
    puts "Create #{credentials_file} with the following:"
    puts "testrail_username: your.username\ntestrail_password: yourpassword"

    exit

  end
end

#load_junit_results(junit_file) ⇒ Hash

Loads the results of a beaker run. Returns hash of failures, passes, and skips that each hold a list of junit xml objects

Examples:

load_junit_results(“~/junit/latest/beaker_junit.xml”)

Parameters:

  • junit_file (String) —

    Path to a junit xml file

Returns:

  • (Hash) —

    A hash containing xml objects for the failures, skips, and passes



256
257
258
259
260
261
262
263
264
 
# File 'lib/telly/test_rail_teller.rb', line 256

def load_junit_results(junit_file)
  junit_doc = Nokogiri::XML(File.read(junit_file))

  failures = junit_doc.xpath('//testcase[failure]')
  skips = junit_doc.xpath('//testcase[skip]')
  passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')

  return {failures: failures, skips: skips, passes: passes}
end

#make_testrail_time(seconds_string) ⇒ String

Returns a string that testrail accepts as an elapsed time Input from beaker is a float in seconds, so it rounds it to the nearest second, and adds an ‘s’ at the end

Testrail throws an exception if it gets “0s”, so it returns a minimum of “1s”

Examples:

puts make_testrail_time(“2.34”) # “2s”

Parameters:

  • seconds_string (String) —

    A string that contains only a number, the elapsed time of a test

Returns:

  • (String) —

    The elapsed time of the test run, rounded and with an ‘s’ appended



233
234
235
236
237
238
239
240
 
# File 'lib/telly/test_rail_teller.rb', line 233

def make_testrail_time(seconds_string)
  # If time is 0, make it 1
  rounded_time = [seconds_string.to_f.round, 1].max
  # Test duration
  time_elapsed = "#{rounded_time}s"

  return time_elapsed
end

#send_to_testrail ⇒ Void

Run the importer

Examples:

password = Telly::main(parse_opts)

Returns:

  • (Void) 


57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/telly/test_rail_teller.rb', line 57

def send_to_testrail()
  options = ArgParser.parse_opts
  # Get pass/fail/skips from junit file
  results = load_junit_results(options[:junit_file])

  puts "Run results:"
  puts "#{results[:passes].length} Passing"
  puts "#{results[:failures].length} Failing or Erroring"
  puts "#{results[:skips].length} Skipped"

  # Set results in testrail
  bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id], options[:dry_run])

  # Print error messages
  if not bad_results.empty?
    puts "Error: There were problems processing these test scripts:"
    bad_results.each do |test_script, error|
      puts "#{test_script}:\n\t#{error}"
    end
  end
end

#set_testrail_results(results, junit_file, testrun_id, dry_run = false) ⇒ Void

Sets the results in testrail. Tests that have testrail API exceptions are kept track of in bad_results

Parameters:

  • results (Hash) —

    A hash of lists of xml objects from the junit output file.

  • junit_file (String) —

    The path to the junit xml file Needed for determining the path of the test file in add_failure, etc

  • testrun_id (String) —

    The TestRail test run ID

  • dry_run (Boolean) (defaults to: false) —

    Do not create api connection when dry_run = true

Returns:

  • (Void) 


129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/telly/test_rail_teller.rb', line 129

def set_testrail_results(results, junit_file, testrun_id, dry_run = false)
  if not dry_run
    credentials = load_credentials(CREDENTIALS_FILE)
    api = get_testrail_api(credentials)
  else
    api = get_testrail_api( { "testrail_username" => 'name', "testrail_password" => 'pass' } )
  end


  # Results that couldn't be set in testrail for some reason
  bad_results = {}

  # passes
  results[:passes].each do |junit_result|
    begin
      submit_result(api, PASSED, junit_result, junit_file, testrun_id, dry_run)
    rescue MissingTestRailId, TestRail::APIError => e
      bad_results[junit_result[:name]] = e.message
    end
  end

  # Failures
  results[:failures].each do |junit_result|
    begin
      submit_result(api, FAILED, junit_result, junit_file, testrun_id, dry_run)
    rescue MissingTestRailId, TestRail::APIError => e
      bad_results[junit_result[:name]] = e.message
    end
  end

  # Skips
  results[:skips].each do |junit_result|
    begin
      submit_result(api, BLOCKED, junit_result, junit_file, testrun_id, dry_run)
    rescue MissingTestRailId, TestRail::APIError => e
      bad_results[junit_result[:name]] = e.message
    end
  end

  return bad_results
end

#submit_result(api, status, junit_result, junit_file, testrun_id, dry_run = false) ⇒ Void

Submits a test result to TestRail

Examples:

submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)

Parameters:

  • api (TestRail::APIClient) —

    TestRail API object

  • status (int) —

    The testrail status to set

  • junit_result (Nokogiri::XML::Element) —

    The nokogiri node that holds the junit result

  • junit_file (String) —

    Path to the junit file the test result originated from

  • testrun_id (String) —

    The testrun ID

  • dry_run (Boolean) (defaults to: false) —

    When true do not send results to TestRail

Returns:

  • (Void)

Raises:

  • (TestRail::APIError) —

    When there is a problem with the API request, testrail raises this exception. Should be caught for error reporting



186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
 
# File 'lib/telly/test_rail_teller.rb', line 186

def submit_result(api, status, junit_result, junit_file, testrun_id, dry_run = false)
  test_file_path = beaker_test_path(junit_file, junit_result)

  puts junit_result.class
  testcase_id = testcase_id_from_beaker_script(test_file_path)

  time_elapsed = make_testrail_time(junit_result[:time])

  # Make appropriate comment for testrail
  case status
  when FAILED
    error_message = junit_result.xpath('./failure').first[:message]
    testrail_comment = "Failed with message:\n#{error_message}"
  when BLOCKED
    skip_message = junit_result.xpath('system-out').first.text
    testrail_comment = "Skipped with message:\n#{skip_message}"
  else
    testrail_comment = "Passed"
  end

  puts "\nSetting result for test case: #{testcase_id}"
  puts "Adding comment:\n#{testrail_comment}"

  if not dry_run
    api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}",
      {
        status_id: status,
        comment: testrail_comment,
        elapsed: time_elapsed,
      }
    )
  end
end

#testcase_id_from_beaker_script(beaker_file) ⇒ String

Extracts the test case id from the test script

Examples:

testcase_id_from_beaker_script(“~/tests/test_the_things.rb”) # 1234

Parameters:

  • beaker_file (String) —

    Path to a beaker test script

Returns:

  • (String) —

    The test case ID

Raises:



274
275
276
277
278
279
280
281
 
# File 'lib/telly/test_rail_teller.rb', line 274

def testcase_id_from_beaker_script(beaker_file)
  # Find first matching line
  match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first

  raise MissingTestRailId, 'Testcase ID could not be found in file' if match.nil?

  match[:testrun_id]
end